Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * Copyright (c) 2010 Red Hat, Inc. All Rights Reserved.
4 : */
5 :
6 : #include "xfs.h"
7 : #include "xfs_fs.h"
8 : #include "xfs_format.h"
9 : #include "xfs_log_format.h"
10 : #include "xfs_shared.h"
11 : #include "xfs_trans_resv.h"
12 : #include "xfs_mount.h"
13 : #include "xfs_extent_busy.h"
14 : #include "xfs_trans.h"
15 : #include "xfs_trans_priv.h"
16 : #include "xfs_log.h"
17 : #include "xfs_log_priv.h"
18 : #include "xfs_trace.h"
19 :
20 : struct workqueue_struct *xfs_discard_wq;
21 :
22 : /*
23 : * Allocate a new ticket. Failing to get a new ticket makes it really hard to
24 : * recover, so we don't allow failure here. Also, we allocate in a context that
25 : * we don't want to be issuing transactions from, so we need to tell the
26 : * allocation code this as well.
27 : *
28 : * We don't reserve any space for the ticket - we are going to steal whatever
29 : * space we require from transactions as they commit. To ensure we reserve all
30 : * the space required, we need to set the current reservation of the ticket to
31 : * zero so that we know to steal the initial transaction overhead from the
32 : * first transaction commit.
33 : */
34 : static struct xlog_ticket *
35 3786727 : xlog_cil_ticket_alloc(
36 : struct xlog *log)
37 : {
38 3786727 : struct xlog_ticket *tic;
39 :
40 3786727 : tic = xlog_ticket_alloc(log, 0, 1, 0);
41 :
42 : /*
43 : * set the current reservation to zero so we know to steal the basic
44 : * transaction overhead reservation from the first transaction commit.
45 : */
46 3786727 : tic->t_curr_res = 0;
47 3786727 : tic->t_iclog_hdrs = 0;
48 3786727 : return tic;
49 : }
50 :
51 : static inline void
52 3811060 : xlog_cil_set_iclog_hdr_count(struct xfs_cil *cil)
53 : {
54 3811060 : struct xlog *log = cil->xc_log;
55 :
56 7622120 : atomic_set(&cil->xc_iclog_hdrs,
57 3811060 : (XLOG_CIL_BLOCKING_SPACE_LIMIT(log) /
58 3811060 : (log->l_iclog_size - log->l_iclog_hsize)));
59 3811060 : }
60 :
61 : /*
62 : * Check if the current log item was first committed in this sequence.
63 : * We can't rely on just the log item being in the CIL, we have to check
64 : * the recorded commit sequence number.
65 : *
66 : * Note: for this to be used in a non-racy manner, it has to be called with
67 : * CIL flushing locked out. As a result, it should only be used during the
68 : * transaction commit process when deciding what to format into the item.
69 : */
70 : static bool
71 2090674454 : xlog_item_in_current_chkpt(
72 : struct xfs_cil *cil,
73 : struct xfs_log_item *lip)
74 : {
75 4181348908 : if (test_bit(XLOG_CIL_EMPTY, &cil->xc_flags))
76 : return false;
77 :
78 : /*
79 : * li_seq is written on the first commit of a log item to record the
80 : * first checkpoint it is written to. Hence if it is different to the
81 : * current sequence, we're in a new checkpoint.
82 : */
83 2090469353 : return lip->li_seq == READ_ONCE(cil->xc_current_sequence);
84 : }
85 :
86 : bool
87 1275472803 : xfs_log_item_in_current_chkpt(
88 : struct xfs_log_item *lip)
89 : {
90 1275472803 : return xlog_item_in_current_chkpt(lip->li_log->l_cilp, lip);
91 : }
92 :
93 : /*
94 : * Unavoidable forward declaration - xlog_cil_push_work() calls
95 : * xlog_cil_ctx_alloc() itself.
96 : */
97 : static void xlog_cil_push_work(struct work_struct *work);
98 :
99 : static struct xfs_cil_ctx *
100 3786735 : xlog_cil_ctx_alloc(void)
101 : {
102 3786735 : struct xfs_cil_ctx *ctx;
103 :
104 3786735 : ctx = kmem_zalloc(sizeof(*ctx), KM_NOFS);
105 3786735 : INIT_LIST_HEAD(&ctx->committing);
106 3786735 : INIT_LIST_HEAD(&ctx->busy_extents);
107 3786735 : INIT_LIST_HEAD(&ctx->log_items);
108 3786735 : INIT_LIST_HEAD(&ctx->lv_chain);
109 3786735 : INIT_WORK(&ctx->push_work, xlog_cil_push_work);
110 3786735 : return ctx;
111 : }
112 :
113 : /*
114 : * Aggregate the CIL per cpu structures into global counts, lists, etc and
115 : * clear the percpu state ready for the next context to use. This is called
116 : * from the push code with the context lock held exclusively, hence nothing else
117 : * will be accessing or modifying the per-cpu counters.
118 : */
119 : static void
120 3762401 : xlog_cil_push_pcp_aggregate(
121 : struct xfs_cil *cil,
122 : struct xfs_cil_ctx *ctx)
123 : {
124 3762401 : struct xlog_cil_pcp *cilpcp;
125 3762401 : int cpu;
126 :
127 11287198 : for_each_cpu_or(cpu, cpu_online_mask, cpu_dying_mask) {
128 7524797 : cilpcp = per_cpu_ptr(cil->xc_pcp, cpu);
129 :
130 7524797 : ctx->ticket->t_curr_res += cilpcp->space_reserved;
131 7524797 : cilpcp->space_reserved = 0;
132 :
133 7524797 : if (!list_empty(&cilpcp->busy_extents)) {
134 3453324 : list_splice_init(&cilpcp->busy_extents,
135 : &ctx->busy_extents);
136 : }
137 7524797 : if (!list_empty(&cilpcp->log_items))
138 6250675 : list_splice_init(&cilpcp->log_items, &ctx->log_items);
139 :
140 : /*
141 : * We're in the middle of switching cil contexts. Reset the
142 : * counter we use to detect when the current context is nearing
143 : * full.
144 : */
145 7524797 : cilpcp->space_used = 0;
146 : }
147 3762402 : }
148 :
149 : /*
150 : * Aggregate the CIL per-cpu space used counters into the global atomic value.
151 : * This is called when the per-cpu counter aggregation will first pass the soft
152 : * limit threshold so we can switch to atomic counter aggregation for accurate
153 : * detection of hard limit traversal.
154 : */
155 : static void
156 918 : xlog_cil_insert_pcp_aggregate(
157 : struct xfs_cil *cil,
158 : struct xfs_cil_ctx *ctx)
159 : {
160 918 : struct xlog_cil_pcp *cilpcp;
161 918 : int cpu;
162 918 : int count = 0;
163 :
164 : /* Trigger atomic updates then aggregate only for the first caller */
165 1836 : if (!test_and_clear_bit(XLOG_CIL_PCP_SPACE, &cil->xc_flags))
166 : return;
167 :
168 2754 : for_each_cpu_or(cpu, cpu_online_mask, cpu_dying_mask) {
169 1836 : int old, prev;
170 :
171 1836 : cilpcp = per_cpu_ptr(cil->xc_pcp, cpu);
172 1836 : do {
173 1836 : old = cilpcp->space_used;
174 1836 : prev = cmpxchg(&cilpcp->space_used, old, 0);
175 1836 : } while (old != prev);
176 1836 : count += old;
177 : }
178 918 : atomic_add(count, &ctx->space_used);
179 : }
180 :
181 : static void
182 3786735 : xlog_cil_ctx_switch(
183 : struct xfs_cil *cil,
184 : struct xfs_cil_ctx *ctx)
185 : {
186 3786735 : xlog_cil_set_iclog_hdr_count(cil);
187 3786735 : set_bit(XLOG_CIL_EMPTY, &cil->xc_flags);
188 3786732 : set_bit(XLOG_CIL_PCP_SPACE, &cil->xc_flags);
189 3786732 : ctx->sequence = ++cil->xc_current_sequence;
190 3786732 : ctx->cil = cil;
191 3786732 : cil->xc_ctx = ctx;
192 3786732 : }
193 :
194 : /*
195 : * After the first stage of log recovery is done, we know where the head and
196 : * tail of the log are. We need this log initialisation done before we can
197 : * initialise the first CIL checkpoint context.
198 : *
199 : * Here we allocate a log ticket to track space usage during a CIL push. This
200 : * ticket is passed to xlog_write() directly so that we don't slowly leak log
201 : * space by failing to account for space used by log headers and additional
202 : * region headers for split regions.
203 : */
204 : void
205 24325 : xlog_cil_init_post_recovery(
206 : struct xlog *log)
207 : {
208 24325 : log->l_cilp->xc_ctx->ticket = xlog_cil_ticket_alloc(log);
209 24325 : log->l_cilp->xc_ctx->sequence = 1;
210 24325 : xlog_cil_set_iclog_hdr_count(log->l_cilp);
211 24325 : }
212 :
213 : static inline int
214 : xlog_cil_iovec_space(
215 : uint niovecs)
216 : {
217 30905985777 : return round_up((sizeof(struct xfs_log_vec) +
218 : niovecs * sizeof(struct xfs_log_iovec)),
219 : sizeof(uint64_t));
220 : }
221 :
222 : /*
223 : * Allocate or pin log vector buffers for CIL insertion.
224 : *
225 : * The CIL currently uses disposable buffers for copying a snapshot of the
226 : * modified items into the log during a push. The biggest problem with this is
227 : * the requirement to allocate the disposable buffer during the commit if:
228 : * a) does not exist; or
229 : * b) it is too small
230 : *
231 : * If we do this allocation within xlog_cil_insert_format_items(), it is done
232 : * under the xc_ctx_lock, which means that a CIL push cannot occur during
233 : * the memory allocation. This means that we have a potential deadlock situation
234 : * under low memory conditions when we have lots of dirty metadata pinned in
235 : * the CIL and we need a CIL commit to occur to free memory.
236 : *
237 : * To avoid this, we need to move the memory allocation outside the
238 : * xc_ctx_lock, but because the log vector buffers are disposable, that opens
239 : * up a TOCTOU race condition w.r.t. the CIL committing and removing the log
240 : * vector buffers between the check and the formatting of the item into the
241 : * log vector buffer within the xc_ctx_lock.
242 : *
243 : * Because the log vector buffer needs to be unchanged during the CIL push
244 : * process, we cannot share the buffer between the transaction commit (which
245 : * modifies the buffer) and the CIL push context that is writing the changes
246 : * into the log. This means skipping preallocation of buffer space is
247 : * unreliable, but we most definitely do not want to be allocating and freeing
248 : * buffers unnecessarily during commits when overwrites can be done safely.
249 : *
250 : * The simplest solution to this problem is to allocate a shadow buffer when a
251 : * log item is committed for the second time, and then to only use this buffer
252 : * if necessary. The buffer can remain attached to the log item until such time
253 : * it is needed, and this is the buffer that is reallocated to match the size of
254 : * the incoming modification. Then during the formatting of the item we can swap
255 : * the active buffer with the new one if we can't reuse the existing buffer. We
256 : * don't free the old buffer as it may be reused on the next modification if
257 : * it's size is right, otherwise we'll free and reallocate it at that point.
258 : *
259 : * This function builds a vector for the changes in each log item in the
260 : * transaction. It then works out the length of the buffer needed for each log
261 : * item, allocates them and attaches the vector to the log item in preparation
262 : * for the formatting step which occurs under the xc_ctx_lock.
263 : *
264 : * While this means the memory footprint goes up, it avoids the repeated
265 : * alloc/free pattern that repeated modifications of an item would otherwise
266 : * cause, and hence minimises the CPU overhead of such behaviour.
267 : */
268 : static void
269 1774899797 : xlog_cil_alloc_shadow_bufs(
270 : struct xlog *log,
271 : struct xfs_trans *tp)
272 : {
273 1774899797 : struct xfs_log_item *lip;
274 :
275 13019189053 : list_for_each_entry(lip, &tp->t_items, li_trans) {
276 11244371288 : struct xfs_log_vec *lv;
277 11244371288 : int niovecs = 0;
278 11244371288 : int nbytes = 0;
279 11244371288 : int buf_size;
280 11244371288 : bool ordered = false;
281 :
282 : /* Skip items which aren't dirty in this transaction. */
283 11244371288 : if (!test_bit(XFS_LI_DIRTY, &lip->li_flags))
284 991773182 : continue;
285 :
286 : /* get number of vecs and size of data to be stored */
287 10252598106 : lip->li_ops->iop_size(lip, &niovecs, &nbytes);
288 :
289 : /*
290 : * Ordered items need to be tracked but we do not wish to write
291 : * them. We need a logvec to track the object, but we do not
292 : * need an iovec or buffer to be allocated for copying data.
293 : */
294 10251788763 : if (niovecs == XFS_LOG_VEC_ORDERED) {
295 1144046 : ordered = true;
296 1144046 : niovecs = 0;
297 1144046 : nbytes = 0;
298 : }
299 :
300 : /*
301 : * We 64-bit align the length of each iovec so that the start of
302 : * the next one is naturally aligned. We'll need to account for
303 : * that slack space here.
304 : *
305 : * We also add the xlog_op_header to each region when
306 : * formatting, but that's not accounted to the size of the item
307 : * at this point. Hence we'll need an addition number of bytes
308 : * for each vector to hold an opheader.
309 : *
310 : * Then round nbytes up to 64-bit alignment so that the initial
311 : * buffer alignment is easy to calculate and verify.
312 : */
313 10251788763 : nbytes += niovecs *
314 : (sizeof(uint64_t) + sizeof(struct xlog_op_header));
315 10251788763 : nbytes = round_up(nbytes, sizeof(uint64_t));
316 :
317 : /*
318 : * The data buffer needs to start 64-bit aligned, so round up
319 : * that space to ensure we can align it appropriately and not
320 : * overrun the buffer.
321 : */
322 10251788763 : buf_size = nbytes + xlog_cil_iovec_space(niovecs);
323 :
324 : /*
325 : * if we have no shadow buffer, or it is too small, we need to
326 : * reallocate it.
327 : */
328 10251788763 : if (!lip->li_lv_shadow ||
329 7497699543 : buf_size > lip->li_lv_shadow->lv_size) {
330 : /*
331 : * We free and allocate here as a realloc would copy
332 : * unnecessary data. We don't use kvzalloc() for the
333 : * same reason - we don't need to zero the data area in
334 : * the buffer, only the log vector header and the iovec
335 : * storage.
336 : */
337 2952292048 : kmem_free(lip->li_lv_shadow);
338 2953012889 : lv = xlog_kvmalloc(buf_size);
339 :
340 2953019359 : memset(lv, 0, xlog_cil_iovec_space(niovecs));
341 :
342 2953019359 : INIT_LIST_HEAD(&lv->lv_list);
343 2953019359 : lv->lv_item = lip;
344 2953019359 : lv->lv_size = buf_size;
345 2953019359 : if (ordered)
346 1113986 : lv->lv_buf_len = XFS_LOG_VEC_ORDERED;
347 : else
348 2951905373 : lv->lv_iovecp = (struct xfs_log_iovec *)&lv[1];
349 2953019359 : lip->li_lv_shadow = lv;
350 : } else {
351 : /* same or smaller, optimise common overwrite case */
352 7299496715 : lv = lip->li_lv_shadow;
353 7299496715 : if (ordered)
354 30062 : lv->lv_buf_len = XFS_LOG_VEC_ORDERED;
355 : else
356 7299466653 : lv->lv_buf_len = 0;
357 7299496715 : lv->lv_bytes = 0;
358 : }
359 :
360 : /* Ensure the lv is set up according to ->iop_size */
361 10252516074 : lv->lv_niovecs = niovecs;
362 :
363 : /* The allocated data region lies beyond the iovec region */
364 10252516074 : lv->lv_buf = (char *)lv + xlog_cil_iovec_space(niovecs);
365 : }
366 :
367 1774817765 : }
368 :
369 : /*
370 : * Prepare the log item for insertion into the CIL. Calculate the difference in
371 : * log space it will consume, and if it is a new item pin it as well.
372 : */
373 : STATIC void
374 9438017592 : xfs_cil_prepare_item(
375 : struct xlog *log,
376 : struct xfs_log_vec *lv,
377 : struct xfs_log_vec *old_lv,
378 : int *diff_len)
379 : {
380 : /* Account for the new LV being passed in */
381 9438017592 : if (lv->lv_buf_len != XFS_LOG_VEC_ORDERED)
382 9436317885 : *diff_len += lv->lv_bytes;
383 :
384 : /*
385 : * If there is no old LV, this is the first time we've seen the item in
386 : * this CIL context and so we need to pin it. If we are replacing the
387 : * old_lv, then remove the space it accounts for and make it the shadow
388 : * buffer for later freeing. In both cases we are now switching to the
389 : * shadow buffer, so update the pointer to it appropriately.
390 : */
391 9438017592 : if (!old_lv) {
392 1853195334 : if (lv->lv_item->li_ops->iop_pin)
393 1033538594 : lv->lv_item->li_ops->iop_pin(lv->lv_item);
394 1853175489 : lv->lv_item->li_lv_shadow = NULL;
395 7584822258 : } else if (old_lv != lv) {
396 135513172 : ASSERT(lv->lv_buf_len != XFS_LOG_VEC_ORDERED);
397 :
398 135513172 : *diff_len -= old_lv->lv_bytes;
399 135513172 : lv->lv_item->li_lv_shadow = old_lv;
400 : }
401 :
402 : /* attach new log vector to log item */
403 9437997747 : lv->lv_item->li_lv = lv;
404 :
405 : /*
406 : * If this is the first time the item is being committed to the
407 : * CIL, store the sequence number on the log item so we can
408 : * tell in future commits whether this is the first checkpoint
409 : * the item is being committed into.
410 : */
411 9437997747 : if (!lv->lv_item->li_seq)
412 1005993707 : lv->lv_item->li_seq = log->l_cilp->xc_ctx->sequence;
413 9437997747 : }
414 :
415 : /*
416 : * Format log item into a flat buffers
417 : *
418 : * For delayed logging, we need to hold a formatted buffer containing all the
419 : * changes on the log item. This enables us to relog the item in memory and
420 : * write it out asynchronously without needing to relock the object that was
421 : * modified at the time it gets written into the iclog.
422 : *
423 : * This function takes the prepared log vectors attached to each log item, and
424 : * formats the changes into the log vector buffer. The buffer it uses is
425 : * dependent on the current state of the vector in the CIL - the shadow lv is
426 : * guaranteed to be large enough for the current modification, but we will only
427 : * use that if we can't reuse the existing lv. If we can't reuse the existing
428 : * lv, then simple swap it out for the shadow lv. We don't free it - that is
429 : * done lazily either by th enext modification or the freeing of the log item.
430 : *
431 : * We don't set up region headers during this process; we simply copy the
432 : * regions into the flat buffer. We can do this because we still have to do a
433 : * formatting step to write the regions into the iclog buffer. Writing the
434 : * ophdrs during the iclog write means that we can support splitting large
435 : * regions across iclog boundares without needing a change in the format of the
436 : * item/region encapsulation.
437 : *
438 : * Hence what we need to do now is change the rewrite the vector array to point
439 : * to the copied region inside the buffer we just allocated. This allows us to
440 : * format the regions into the iclog as though they are being formatted
441 : * directly out of the objects themselves.
442 : */
443 : static void
444 1774884669 : xlog_cil_insert_format_items(
445 : struct xlog *log,
446 : struct xfs_trans *tp,
447 : int *diff_len)
448 : {
449 1774884669 : struct xfs_log_item *lip;
450 :
451 : /* Bail out if we didn't find a log item. */
452 1774884669 : if (list_empty(&tp->t_items)) {
453 0 : ASSERT(0);
454 0 : return;
455 : }
456 :
457 12203553814 : list_for_each_entry(lip, &tp->t_items, li_trans) {
458 10428752078 : struct xfs_log_vec *lv;
459 10428752078 : struct xfs_log_vec *old_lv = NULL;
460 10428752078 : struct xfs_log_vec *shadow;
461 10428752078 : bool ordered = false;
462 :
463 : /* Skip items which aren't dirty in this transaction. */
464 10428752078 : if (!test_bit(XFS_LI_DIRTY, &lip->li_flags))
465 991774365 : continue;
466 :
467 : /*
468 : * The formatting size information is already attached to
469 : * the shadow lv on the log item.
470 : */
471 9436977713 : shadow = lip->li_lv_shadow;
472 9436977713 : if (shadow->lv_buf_len == XFS_LOG_VEC_ORDERED)
473 1144047 : ordered = true;
474 :
475 : /* Skip items that do not have any vectors for writing */
476 9436977713 : if (!shadow->lv_niovecs && !ordered)
477 0 : continue;
478 :
479 : /* compare to existing item size */
480 9436977713 : old_lv = lip->li_lv;
481 9436977713 : if (lip->li_lv && shadow->lv_size <= lip->li_lv->lv_size) {
482 : /* same or smaller, optimise common overwrite case */
483 7448720745 : lv = lip->li_lv;
484 :
485 7448720745 : if (ordered)
486 59164 : goto insert;
487 :
488 : /*
489 : * set the item up as though it is a new insertion so
490 : * that the space reservation accounting is correct.
491 : */
492 7448661581 : *diff_len -= lv->lv_bytes;
493 :
494 : /* Ensure the lv is set up according to ->iop_size */
495 7448661581 : lv->lv_niovecs = shadow->lv_niovecs;
496 :
497 : /* reset the lv buffer information for new formatting */
498 7448661581 : lv->lv_buf_len = 0;
499 7448661581 : lv->lv_bytes = 0;
500 7448661581 : lv->lv_buf = (char *)lv +
501 7448661581 : xlog_cil_iovec_space(lv->lv_niovecs);
502 : } else {
503 : /* switch to shadow buffer! */
504 1988256968 : lv = shadow;
505 1988256968 : lv->lv_item = lip;
506 1988256968 : if (ordered) {
507 : /* track as an ordered logvec */
508 1084891 : ASSERT(lip->li_lv == NULL);
509 1084891 : goto insert;
510 : }
511 : }
512 :
513 9435833658 : ASSERT(IS_ALIGNED((unsigned long)lv->lv_buf, sizeof(uint64_t)));
514 9435833658 : lip->li_ops->iop_format(lip, lv);
515 9437612823 : insert:
516 9437612823 : xfs_cil_prepare_item(log, lv, old_lv, diff_len);
517 : }
518 : }
519 :
520 : /*
521 : * The use of lockless waitqueue_active() requires that the caller has
522 : * serialised itself against the wakeup call in xlog_cil_push_work(). That
523 : * can be done by either holding the push lock or the context lock.
524 : */
525 : static inline bool
526 1636068773 : xlog_cil_over_hard_limit(
527 : struct xlog *log,
528 : int32_t space_used)
529 : {
530 1636068773 : if (waitqueue_active(&log->l_cilp->xc_push_wait))
531 : return true;
532 1636083791 : if (space_used >= XLOG_CIL_BLOCKING_SPACE_LIMIT(log))
533 0 : return true;
534 : return false;
535 : }
536 :
537 : /*
538 : * Insert the log items into the CIL and calculate the difference in space
539 : * consumed by the item. Add the space to the checkpoint ticket and calculate
540 : * if the change requires additional log metadata. If it does, take that space
541 : * as well. Remove the amount of space we added to the checkpoint ticket from
542 : * the current transaction ticket so that the accounting works out correctly.
543 : */
544 : static void
545 1774871424 : xlog_cil_insert_items(
546 : struct xlog *log,
547 : struct xfs_trans *tp,
548 : uint32_t released_space)
549 : {
550 1774871424 : struct xfs_cil *cil = log->l_cilp;
551 1774871424 : struct xfs_cil_ctx *ctx = cil->xc_ctx;
552 1774871424 : struct xfs_log_item *lip;
553 1774871424 : int len = 0;
554 1774871424 : int iovhdr_res = 0, split_res = 0, ctx_res = 0;
555 1774871424 : int space_used;
556 1774871424 : int order;
557 1774871424 : struct xlog_cil_pcp *cilpcp;
558 :
559 1774871424 : ASSERT(tp);
560 :
561 : /*
562 : * We can do this safely because the context can't checkpoint until we
563 : * are done so it doesn't matter exactly how we update the CIL.
564 : */
565 1774871424 : xlog_cil_insert_format_items(log, tp, &len);
566 :
567 : /*
568 : * Subtract the space released by intent cancelation from the space we
569 : * consumed so that we remove it from the CIL space and add it back to
570 : * the current transaction reservation context.
571 : */
572 1774842059 : len -= released_space;
573 :
574 : /*
575 : * Grab the per-cpu pointer for the CIL before we start any accounting.
576 : * That ensures that we are running with pre-emption disabled and so we
577 : * can't be scheduled away between split sample/update operations that
578 : * are done without outside locking to serialise them.
579 : */
580 1774842059 : cilpcp = get_cpu_ptr(cil->xc_pcp);
581 :
582 : /*
583 : * We need to take the CIL checkpoint unit reservation on the first
584 : * commit into the CIL. Test the XLOG_CIL_EMPTY bit first so we don't
585 : * unnecessarily do an atomic op in the fast path here. We can clear the
586 : * XLOG_CIL_EMPTY bit as we are under the xc_ctx_lock here and that
587 : * needs to be held exclusively to reset the XLOG_CIL_EMPTY bit.
588 : */
589 3553545686 : if (test_bit(XLOG_CIL_EMPTY, &cil->xc_flags) &&
590 : test_and_clear_bit(XLOG_CIL_EMPTY, &cil->xc_flags))
591 3762395 : ctx_res = ctx->ticket->t_unit_res;
592 :
593 : /*
594 : * Check if we need to steal iclog headers. atomic_read() is not a
595 : * locked atomic operation, so we can check the value before we do any
596 : * real atomic ops in the fast path. If we've already taken the CIL unit
597 : * reservation from this commit, we've already got one iclog header
598 : * space reserved so we have to account for that otherwise we risk
599 : * overrunning the reservation on this ticket.
600 : *
601 : * If the CIL is already at the hard limit, we might need more header
602 : * space that originally reserved. So steal more header space from every
603 : * commit that occurs once we are over the hard limit to ensure the CIL
604 : * push won't run out of reservation space.
605 : *
606 : * This can steal more than we need, but that's OK.
607 : *
608 : * The cil->xc_ctx_lock provides the serialisation necessary for safely
609 : * calling xlog_cil_over_hard_limit() in this context.
610 : */
611 1774891536 : space_used = atomic_read(&ctx->space_used) + cilpcp->space_used + len;
612 3410963221 : if (atomic_read(&cil->xc_iclog_hdrs) > 0 ||
613 1636083439 : xlog_cil_over_hard_limit(log, space_used)) {
614 138808097 : split_res = log->l_iclog_hsize +
615 : sizeof(struct xlog_op_header);
616 138808097 : if (ctx_res)
617 3762394 : ctx_res += split_res * (tp->t_ticket->t_iclog_hdrs - 1);
618 : else
619 135045703 : ctx_res = split_res * tp->t_ticket->t_iclog_hdrs;
620 138808097 : atomic_sub(tp->t_ticket->t_iclog_hdrs, &cil->xc_iclog_hdrs);
621 : }
622 1774937072 : cilpcp->space_reserved += ctx_res;
623 :
624 : /*
625 : * Accurately account when over the soft limit, otherwise fold the
626 : * percpu count into the global count if over the per-cpu threshold.
627 : */
628 1774937072 : if (!test_bit(XLOG_CIL_PCP_SPACE, &cil->xc_flags)) {
629 4050 : atomic_add(len, &ctx->space_used);
630 1774933022 : } else if (cilpcp->space_used + len >
631 1774933028 : (XLOG_CIL_SPACE_LIMIT(log) / num_online_cpus())) {
632 289121 : space_used = atomic_add_return(cilpcp->space_used + len,
633 : &ctx->space_used);
634 289121 : cilpcp->space_used = 0;
635 :
636 : /*
637 : * If we just transitioned over the soft limit, we need to
638 : * transition to the global atomic counter.
639 : */
640 289121 : if (space_used >= XLOG_CIL_SPACE_LIMIT(log))
641 918 : xlog_cil_insert_pcp_aggregate(cil, ctx);
642 : } else {
643 1774643901 : cilpcp->space_used += len;
644 : }
645 : /* attach the transaction to the CIL if it has any busy extents */
646 1774937072 : if (!list_empty(&tp->t_busy))
647 50934650 : list_splice_init(&tp->t_busy, &cilpcp->busy_extents);
648 :
649 : /*
650 : * Now update the order of everything modified in the transaction
651 : * and insert items into the CIL if they aren't already there.
652 : * We do this here so we only need to take the CIL lock once during
653 : * the transaction commit.
654 : */
655 1774937072 : order = atomic_inc_return(&ctx->order_id);
656 12205202600 : list_for_each_entry(lip, &tp->t_items, li_trans) {
657 : /* Skip items which aren't dirty in this transaction. */
658 10430250031 : if (!test_bit(XFS_LI_DIRTY, &lip->li_flags))
659 991776544 : continue;
660 :
661 9438473487 : lip->li_order_id = order;
662 9438473487 : if (!list_empty(&lip->li_cil))
663 7585263261 : continue;
664 1853210226 : list_add_tail(&lip->li_cil, &cilpcp->log_items);
665 : }
666 1774952569 : put_cpu_ptr(cilpcp);
667 :
668 : /*
669 : * If we've overrun the reservation, dump the tx details before we move
670 : * the log items. Shutdown is imminent...
671 : */
672 1774961229 : tp->t_ticket->t_curr_res -= ctx_res + len;
673 1774961229 : if (WARN_ON(tp->t_ticket->t_curr_res < 0)) {
674 0 : xfs_warn(log->l_mp, "Transaction log reservation overrun:");
675 0 : xfs_warn(log->l_mp,
676 : " log items: %d bytes (iov hdrs: %d bytes)",
677 : len, iovhdr_res);
678 0 : xfs_warn(log->l_mp, " split region headers: %d bytes",
679 : split_res);
680 0 : xfs_warn(log->l_mp, " ctx ticket: %d bytes", ctx_res);
681 0 : xlog_print_trans(tp);
682 0 : xlog_force_shutdown(log, SHUTDOWN_LOG_IO_ERROR);
683 : }
684 1774961229 : }
685 :
686 : static void
687 3762402 : xlog_cil_free_logvec(
688 : struct list_head *lv_chain)
689 : {
690 3762402 : struct xfs_log_vec *lv;
691 :
692 1042219941 : while (!list_empty(lv_chain)) {
693 1038457539 : lv = list_first_entry(lv_chain, struct xfs_log_vec, lv_list);
694 1038457539 : list_del_init(&lv->lv_list);
695 1038457527 : kmem_free(lv);
696 : }
697 3762402 : }
698 :
699 : static void
700 61 : xlog_discard_endio_work(
701 : struct work_struct *work)
702 : {
703 61 : struct xfs_cil_ctx *ctx =
704 61 : container_of(work, struct xfs_cil_ctx, discard_endio_work);
705 61 : struct xfs_mount *mp = ctx->cil->xc_log->l_mp;
706 :
707 61 : xfs_extent_busy_clear(mp, &ctx->busy_extents, false);
708 61 : kmem_free(ctx);
709 61 : }
710 :
711 : /*
712 : * Queue up the actual completion to a thread to avoid IRQ-safe locking for
713 : * pagb_lock. Note that we need a unbounded workqueue, otherwise we might
714 : * get the execution delayed up to 30 seconds for weird reasons.
715 : */
716 : static void
717 61 : xlog_discard_endio(
718 : struct bio *bio)
719 : {
720 61 : struct xfs_cil_ctx *ctx = bio->bi_private;
721 :
722 61 : INIT_WORK(&ctx->discard_endio_work, xlog_discard_endio_work);
723 61 : queue_work(xfs_discard_wq, &ctx->discard_endio_work);
724 61 : bio_put(bio);
725 61 : }
726 :
727 : static void
728 61 : xlog_discard_busy_extents(
729 : struct xfs_mount *mp,
730 : struct xfs_cil_ctx *ctx)
731 : {
732 61 : struct list_head *list = &ctx->busy_extents;
733 61 : struct xfs_extent_busy *busyp;
734 61 : struct bio *bio = NULL;
735 61 : struct blk_plug plug;
736 61 : int error = 0;
737 :
738 61 : ASSERT(xfs_has_discard(mp));
739 :
740 61 : blk_start_plug(&plug);
741 330 : list_for_each_entry(busyp, list, list) {
742 269 : trace_xfs_discard_extent(mp, busyp->agno, busyp->bno,
743 : busyp->length);
744 :
745 538 : error = __blkdev_issue_discard(
746 269 : xfs_buftarg_bdev(mp->m_ddev_targp),
747 269 : XFS_AGB_TO_DADDR(mp, busyp->agno, busyp->bno),
748 269 : XFS_FSB_TO_BB(mp, busyp->length),
749 : GFP_NOFS, &bio);
750 269 : if (error && error != -EOPNOTSUPP) {
751 0 : xfs_info(mp,
752 : "discard failed for extent [0x%llx,%u], error %d",
753 : (unsigned long long)busyp->bno,
754 : busyp->length,
755 : error);
756 0 : break;
757 : }
758 : }
759 :
760 61 : if (bio) {
761 61 : bio->bi_private = ctx;
762 61 : bio->bi_end_io = xlog_discard_endio;
763 61 : submit_bio(bio);
764 : } else {
765 0 : xlog_discard_endio_work(&ctx->discard_endio_work);
766 : }
767 61 : blk_finish_plug(&plug);
768 61 : }
769 :
770 : /*
771 : * Mark all items committed and clear busy extents. We free the log vector
772 : * chains in a separate pass so that we unpin the log items as quickly as
773 : * possible.
774 : */
775 : static void
776 3762402 : xlog_cil_committed(
777 : struct xfs_cil_ctx *ctx)
778 : {
779 3762402 : struct xfs_mount *mp = ctx->cil->xc_log->l_mp;
780 3762402 : bool abort = xlog_is_shutdown(ctx->cil->xc_log);
781 :
782 : /*
783 : * If the I/O failed, we're aborting the commit and already shutdown.
784 : * Wake any commit waiters before aborting the log items so we don't
785 : * block async log pushers on callbacks. Async log pushers explicitly do
786 : * not wait on log force completion because they may be holding locks
787 : * required to unpin items.
788 : */
789 3762402 : if (abort) {
790 11307 : spin_lock(&ctx->cil->xc_push_lock);
791 11307 : wake_up_all(&ctx->cil->xc_start_wait);
792 11307 : wake_up_all(&ctx->cil->xc_commit_wait);
793 11307 : spin_unlock(&ctx->cil->xc_push_lock);
794 : }
795 :
796 3762402 : xfs_trans_committed_bulk(ctx->cil->xc_log->l_ailp, &ctx->lv_chain,
797 : ctx->start_lsn, abort);
798 :
799 3762402 : xfs_extent_busy_sort(&ctx->busy_extents);
800 7524804 : xfs_extent_busy_clear(mp, &ctx->busy_extents,
801 3762402 : xfs_has_discard(mp) && !abort);
802 :
803 3762402 : spin_lock(&ctx->cil->xc_push_lock);
804 3762402 : list_del(&ctx->committing);
805 3762402 : spin_unlock(&ctx->cil->xc_push_lock);
806 :
807 3762402 : xlog_cil_free_logvec(&ctx->lv_chain);
808 :
809 3762402 : if (!list_empty(&ctx->busy_extents))
810 61 : xlog_discard_busy_extents(mp, ctx);
811 : else
812 3762341 : kmem_free(ctx);
813 3762402 : }
814 :
815 : void
816 32419798 : xlog_cil_process_committed(
817 : struct list_head *list)
818 : {
819 32419798 : struct xfs_cil_ctx *ctx;
820 :
821 36175038 : while ((ctx = list_first_entry_or_null(list,
822 : struct xfs_cil_ctx, iclog_entry))) {
823 3755240 : list_del(&ctx->iclog_entry);
824 3755240 : xlog_cil_committed(ctx);
825 : }
826 32419798 : }
827 :
828 : /*
829 : * Record the LSN of the iclog we were just granted space to start writing into.
830 : * If the context doesn't have a start_lsn recorded, then this iclog will
831 : * contain the start record for the checkpoint. Otherwise this write contains
832 : * the commit record for the checkpoint.
833 : */
834 : void
835 7513479 : xlog_cil_set_ctx_write_state(
836 : struct xfs_cil_ctx *ctx,
837 : struct xlog_in_core *iclog)
838 : {
839 7513479 : struct xfs_cil *cil = ctx->cil;
840 7513479 : xfs_lsn_t lsn = be64_to_cpu(iclog->ic_header.h_lsn);
841 :
842 7513479 : ASSERT(!ctx->commit_lsn);
843 7513479 : if (!ctx->start_lsn) {
844 3758248 : spin_lock(&cil->xc_push_lock);
845 : /*
846 : * The LSN we need to pass to the log items on transaction
847 : * commit is the LSN reported by the first log vector write, not
848 : * the commit lsn. If we use the commit record lsn then we can
849 : * move the grant write head beyond the tail LSN and overwrite
850 : * it.
851 : */
852 3758249 : ctx->start_lsn = lsn;
853 3758249 : wake_up_all(&cil->xc_start_wait);
854 3758250 : spin_unlock(&cil->xc_push_lock);
855 :
856 : /*
857 : * Make sure the metadata we are about to overwrite in the log
858 : * has been flushed to stable storage before this iclog is
859 : * issued.
860 : */
861 3758249 : spin_lock(&cil->xc_log->l_icloglock);
862 3758250 : iclog->ic_flags |= XLOG_ICL_NEED_FLUSH;
863 3758250 : spin_unlock(&cil->xc_log->l_icloglock);
864 3758250 : return;
865 : }
866 :
867 : /*
868 : * Take a reference to the iclog for the context so that we still hold
869 : * it when xlog_write is done and has released it. This means the
870 : * context controls when the iclog is released for IO.
871 : */
872 3755231 : atomic_inc(&iclog->ic_refcnt);
873 :
874 : /*
875 : * xlog_state_get_iclog_space() guarantees there is enough space in the
876 : * iclog for an entire commit record, so we can attach the context
877 : * callbacks now. This needs to be done before we make the commit_lsn
878 : * visible to waiters so that checkpoints with commit records in the
879 : * same iclog order their IO completion callbacks in the same order that
880 : * the commit records appear in the iclog.
881 : */
882 3755240 : spin_lock(&cil->xc_log->l_icloglock);
883 3755240 : list_add_tail(&ctx->iclog_entry, &iclog->ic_callbacks);
884 3755240 : spin_unlock(&cil->xc_log->l_icloglock);
885 :
886 : /*
887 : * Now we can record the commit LSN and wake anyone waiting for this
888 : * sequence to have the ordered commit record assigned to a physical
889 : * location in the log.
890 : */
891 3755240 : spin_lock(&cil->xc_push_lock);
892 3755240 : ctx->commit_iclog = iclog;
893 3755240 : ctx->commit_lsn = lsn;
894 3755240 : wake_up_all(&cil->xc_commit_wait);
895 3755240 : spin_unlock(&cil->xc_push_lock);
896 : }
897 :
898 :
899 : /*
900 : * Ensure that the order of log writes follows checkpoint sequence order. This
901 : * relies on the context LSN being zero until the log write has guaranteed the
902 : * LSN that the log write will start at via xlog_state_get_iclog_space().
903 : */
904 : enum _record_type {
905 : _START_RECORD,
906 : _COMMIT_RECORD,
907 : };
908 :
909 : static int
910 7517636 : xlog_cil_order_write(
911 : struct xfs_cil *cil,
912 : xfs_csn_t sequence,
913 : enum _record_type record)
914 : {
915 7528838 : struct xfs_cil_ctx *ctx;
916 :
917 : restart:
918 7528838 : spin_lock(&cil->xc_push_lock);
919 16158720 : list_for_each_entry(ctx, &cil->xc_committing, committing) {
920 : /*
921 : * Avoid getting stuck in this loop because we were woken by the
922 : * shutdown, but then went back to sleep once already in the
923 : * shutdown state.
924 : */
925 17290468 : if (xlog_is_shutdown(cil->xc_log)) {
926 4152 : spin_unlock(&cil->xc_push_lock);
927 4152 : return -EIO;
928 : }
929 :
930 : /*
931 : * Higher sequences will wait for this one so skip them.
932 : * Don't wait for our own sequence, either.
933 : */
934 8641082 : if (ctx->sequence >= sequence)
935 7542247 : continue;
936 :
937 : /* Wait until the LSN for the record has been recorded. */
938 1098835 : switch (record) {
939 710369 : case _START_RECORD:
940 710369 : if (!ctx->start_lsn) {
941 660 : xlog_wait(&cil->xc_start_wait, &cil->xc_push_lock);
942 660 : goto restart;
943 : }
944 : break;
945 388466 : case _COMMIT_RECORD:
946 388466 : if (!ctx->commit_lsn) {
947 10542 : xlog_wait(&cil->xc_commit_wait, &cil->xc_push_lock);
948 10542 : goto restart;
949 : }
950 : break;
951 : }
952 : }
953 7513486 : spin_unlock(&cil->xc_push_lock);
954 7513486 : return 0;
955 : }
956 :
957 : /*
958 : * Write out the log vector change now attached to the CIL context. This will
959 : * write a start record that needs to be strictly ordered in ascending CIL
960 : * sequence order so that log recovery will always use in-order start LSNs when
961 : * replaying checkpoints.
962 : */
963 : static int
964 3762400 : xlog_cil_write_chain(
965 : struct xfs_cil_ctx *ctx,
966 : uint32_t chain_len)
967 : {
968 3762400 : struct xlog *log = ctx->cil->xc_log;
969 3762400 : int error;
970 :
971 3762400 : error = xlog_cil_order_write(ctx->cil, ctx->sequence, _START_RECORD);
972 3762401 : if (error)
973 : return error;
974 3758249 : return xlog_write(log, ctx, &ctx->lv_chain, ctx->ticket, chain_len);
975 : }
976 :
977 : /*
978 : * Write out the commit record of a checkpoint transaction to close off a
979 : * running log write. These commit records are strictly ordered in ascending CIL
980 : * sequence order so that log recovery will always replay the checkpoints in the
981 : * correct order.
982 : */
983 : static int
984 3755240 : xlog_cil_write_commit_record(
985 : struct xfs_cil_ctx *ctx)
986 : {
987 3755240 : struct xlog *log = ctx->cil->xc_log;
988 7510480 : struct xlog_op_header ophdr = {
989 : .oh_clientid = XFS_TRANSACTION,
990 3755240 : .oh_tid = cpu_to_be32(ctx->ticket->t_tid),
991 : .oh_flags = XLOG_COMMIT_TRANS,
992 : };
993 3755240 : struct xfs_log_iovec reg = {
994 : .i_addr = &ophdr,
995 : .i_len = sizeof(struct xlog_op_header),
996 : .i_type = XLOG_REG_TYPE_COMMIT,
997 : };
998 3755240 : struct xfs_log_vec vec = {
999 : .lv_niovecs = 1,
1000 : .lv_iovecp = ®,
1001 : };
1002 3755240 : int error;
1003 3755240 : LIST_HEAD(lv_chain);
1004 3755240 : list_add(&vec.lv_list, &lv_chain);
1005 :
1006 7510476 : if (xlog_is_shutdown(log))
1007 : return -EIO;
1008 :
1009 3755238 : error = xlog_cil_order_write(ctx->cil, ctx->sequence, _COMMIT_RECORD);
1010 3755240 : if (error)
1011 : return error;
1012 :
1013 : /* account for space used by record data */
1014 3755240 : ctx->ticket->t_curr_res -= reg.i_len;
1015 3755240 : error = xlog_write(log, ctx, &lv_chain, ctx->ticket, reg.i_len);
1016 3755240 : if (error)
1017 0 : xlog_force_shutdown(log, SHUTDOWN_LOG_IO_ERROR);
1018 : return error;
1019 : }
1020 :
1021 : struct xlog_cil_trans_hdr {
1022 : struct xlog_op_header oph[2];
1023 : struct xfs_trans_header thdr;
1024 : struct xfs_log_iovec lhdr[2];
1025 : };
1026 :
1027 : /*
1028 : * Build a checkpoint transaction header to begin the journal transaction. We
1029 : * need to account for the space used by the transaction header here as it is
1030 : * not accounted for in xlog_write().
1031 : *
1032 : * This is the only place we write a transaction header, so we also build the
1033 : * log opheaders that indicate the start of a log transaction and wrap the
1034 : * transaction header. We keep the start record in it's own log vector rather
1035 : * than compacting them into a single region as this ends up making the logic
1036 : * in xlog_write() for handling empty opheaders for start, commit and unmount
1037 : * records much simpler.
1038 : */
1039 : static void
1040 3762401 : xlog_cil_build_trans_hdr(
1041 : struct xfs_cil_ctx *ctx,
1042 : struct xlog_cil_trans_hdr *hdr,
1043 : struct xfs_log_vec *lvhdr,
1044 : int num_iovecs)
1045 : {
1046 3762401 : struct xlog_ticket *tic = ctx->ticket;
1047 3762401 : __be32 tid = cpu_to_be32(tic->t_tid);
1048 :
1049 3762401 : memset(hdr, 0, sizeof(*hdr));
1050 :
1051 : /* Log start record */
1052 3762401 : hdr->oph[0].oh_tid = tid;
1053 3762401 : hdr->oph[0].oh_clientid = XFS_TRANSACTION;
1054 3762401 : hdr->oph[0].oh_flags = XLOG_START_TRANS;
1055 :
1056 : /* log iovec region pointer */
1057 3762401 : hdr->lhdr[0].i_addr = &hdr->oph[0];
1058 3762401 : hdr->lhdr[0].i_len = sizeof(struct xlog_op_header);
1059 3762401 : hdr->lhdr[0].i_type = XLOG_REG_TYPE_LRHEADER;
1060 :
1061 : /* log opheader */
1062 3762401 : hdr->oph[1].oh_tid = tid;
1063 3762401 : hdr->oph[1].oh_clientid = XFS_TRANSACTION;
1064 3762401 : hdr->oph[1].oh_len = cpu_to_be32(sizeof(struct xfs_trans_header));
1065 :
1066 : /* transaction header in host byte order format */
1067 3762401 : hdr->thdr.th_magic = XFS_TRANS_HEADER_MAGIC;
1068 3762401 : hdr->thdr.th_type = XFS_TRANS_CHECKPOINT;
1069 3762401 : hdr->thdr.th_tid = tic->t_tid;
1070 3762401 : hdr->thdr.th_num_items = num_iovecs;
1071 :
1072 : /* log iovec region pointer */
1073 3762401 : hdr->lhdr[1].i_addr = &hdr->oph[1];
1074 3762401 : hdr->lhdr[1].i_len = sizeof(struct xlog_op_header) +
1075 : sizeof(struct xfs_trans_header);
1076 3762401 : hdr->lhdr[1].i_type = XLOG_REG_TYPE_TRANSHDR;
1077 :
1078 3762401 : lvhdr->lv_niovecs = 2;
1079 3762401 : lvhdr->lv_iovecp = &hdr->lhdr[0];
1080 3762401 : lvhdr->lv_bytes = hdr->lhdr[0].i_len + hdr->lhdr[1].i_len;
1081 :
1082 3762401 : tic->t_curr_res -= lvhdr->lv_bytes;
1083 3762401 : }
1084 :
1085 : /*
1086 : * CIL item reordering compare function. We want to order in ascending ID order,
1087 : * but we want to leave items with the same ID in the order they were added to
1088 : * the list. This is important for operations like reflink where we log 4 order
1089 : * dependent intents in a single transaction when we overwrite an existing
1090 : * shared extent with a new shared extent. i.e. BUI(unmap), CUI(drop),
1091 : * CUI (inc), BUI(remap)...
1092 : */
1093 : static int
1094 7594947206 : xlog_cil_order_cmp(
1095 : void *priv,
1096 : const struct list_head *a,
1097 : const struct list_head *b)
1098 : {
1099 7594947206 : struct xfs_log_vec *l1 = container_of(a, struct xfs_log_vec, lv_list);
1100 7594947206 : struct xfs_log_vec *l2 = container_of(b, struct xfs_log_vec, lv_list);
1101 :
1102 7594947206 : return l1->lv_order_id > l2->lv_order_id;
1103 : }
1104 :
1105 : /*
1106 : * Pull all the log vectors off the items in the CIL, and remove the items from
1107 : * the CIL. We don't need the CIL lock here because it's only needed on the
1108 : * transaction commit side which is currently locked out by the flush lock.
1109 : *
1110 : * If a log item is marked with a whiteout, we do not need to write it to the
1111 : * journal and so we just move them to the whiteout list for the caller to
1112 : * dispose of appropriately.
1113 : */
1114 : static void
1115 3762402 : xlog_cil_build_lv_chain(
1116 : struct xfs_cil_ctx *ctx,
1117 : struct list_head *whiteouts,
1118 : uint32_t *num_iovecs,
1119 : uint32_t *num_bytes)
1120 : {
1121 1856992826 : while (!list_empty(&ctx->log_items)) {
1122 1853230425 : struct xfs_log_item *item;
1123 1853230425 : struct xfs_log_vec *lv;
1124 :
1125 1853230425 : item = list_first_entry(&ctx->log_items,
1126 : struct xfs_log_item, li_cil);
1127 :
1128 3706460850 : if (test_bit(XFS_LI_WHITEOUT, &item->li_flags)) {
1129 814774451 : list_move(&item->li_cil, whiteouts);
1130 814774644 : trace_xfs_cil_whiteout_skip(item);
1131 814774581 : continue;
1132 : }
1133 :
1134 1038455974 : lv = item->li_lv;
1135 1038455974 : lv->lv_order_id = item->li_order_id;
1136 :
1137 : /* we don't write ordered log vectors */
1138 1038455974 : if (lv->lv_buf_len != XFS_LOG_VEC_ORDERED)
1139 1037426781 : *num_bytes += lv->lv_bytes;
1140 1038455974 : *num_iovecs += lv->lv_niovecs;
1141 1038455974 : list_add_tail(&lv->lv_list, &ctx->lv_chain);
1142 :
1143 1038455945 : list_del_init(&item->li_cil);
1144 1038455843 : item->li_order_id = 0;
1145 1038455843 : item->li_lv = NULL;
1146 : }
1147 3762401 : }
1148 :
1149 : static void
1150 3762401 : xlog_cil_cleanup_whiteouts(
1151 : struct list_head *whiteouts)
1152 : {
1153 818536460 : while (!list_empty(whiteouts)) {
1154 814774061 : struct xfs_log_item *item = list_first_entry(whiteouts,
1155 : struct xfs_log_item, li_cil);
1156 814774061 : list_del_init(&item->li_cil);
1157 814768642 : trace_xfs_cil_whiteout_unpin(item);
1158 814768762 : item->li_ops->iop_unpin(item, 1);
1159 : }
1160 3762399 : }
1161 :
1162 : /*
1163 : * Push the Committed Item List to the log.
1164 : *
1165 : * If the current sequence is the same as xc_push_seq we need to do a flush. If
1166 : * xc_push_seq is less than the current sequence, then it has already been
1167 : * flushed and we don't need to do anything - the caller will wait for it to
1168 : * complete if necessary.
1169 : *
1170 : * xc_push_seq is checked unlocked against the sequence number for a match.
1171 : * Hence we can allow log forces to run racily and not issue pushes for the
1172 : * same sequence twice. If we get a race between multiple pushes for the same
1173 : * sequence they will block on the first one and then abort, hence avoiding
1174 : * needless pushes.
1175 : */
1176 : static void
1177 3762402 : xlog_cil_push_work(
1178 : struct work_struct *work)
1179 : {
1180 3762402 : struct xfs_cil_ctx *ctx =
1181 3762402 : container_of(work, struct xfs_cil_ctx, push_work);
1182 3762402 : struct xfs_cil *cil = ctx->cil;
1183 3762402 : struct xlog *log = cil->xc_log;
1184 3762402 : struct xfs_cil_ctx *new_ctx;
1185 3762402 : int num_iovecs = 0;
1186 3762402 : int num_bytes = 0;
1187 3762402 : int error = 0;
1188 3762402 : struct xlog_cil_trans_hdr thdr;
1189 3762402 : struct xfs_log_vec lvhdr = {};
1190 3762402 : xfs_csn_t push_seq;
1191 3762402 : bool push_commit_stable;
1192 3762402 : LIST_HEAD (whiteouts);
1193 3762402 : struct xlog_ticket *ticket;
1194 :
1195 3762402 : new_ctx = xlog_cil_ctx_alloc();
1196 3762402 : new_ctx->ticket = xlog_cil_ticket_alloc(log);
1197 :
1198 3762402 : down_write(&cil->xc_ctx_lock);
1199 :
1200 3762402 : spin_lock(&cil->xc_push_lock);
1201 3762402 : push_seq = cil->xc_push_seq;
1202 3762402 : ASSERT(push_seq <= ctx->sequence);
1203 3762402 : push_commit_stable = cil->xc_push_commit_stable;
1204 3762402 : cil->xc_push_commit_stable = false;
1205 :
1206 : /*
1207 : * As we are about to switch to a new, empty CIL context, we no longer
1208 : * need to throttle tasks on CIL space overruns. Wake any waiters that
1209 : * the hard push throttle may have caught so they can start committing
1210 : * to the new context. The ctx->xc_push_lock provides the serialisation
1211 : * necessary for safely using the lockless waitqueue_active() check in
1212 : * this context.
1213 : */
1214 3762402 : if (waitqueue_active(&cil->xc_push_wait))
1215 0 : wake_up_all(&cil->xc_push_wait);
1216 :
1217 3762402 : xlog_cil_push_pcp_aggregate(cil, ctx);
1218 :
1219 : /*
1220 : * Check if we've anything to push. If there is nothing, then we don't
1221 : * move on to a new sequence number and so we have to be able to push
1222 : * this sequence again later.
1223 : */
1224 7524802 : if (test_bit(XLOG_CIL_EMPTY, &cil->xc_flags)) {
1225 0 : cil->xc_push_seq = 0;
1226 0 : spin_unlock(&cil->xc_push_lock);
1227 0 : goto out_skip;
1228 : }
1229 :
1230 :
1231 : /* check for a previously pushed sequence */
1232 3762401 : if (push_seq < ctx->sequence) {
1233 0 : spin_unlock(&cil->xc_push_lock);
1234 0 : goto out_skip;
1235 : }
1236 :
1237 : /*
1238 : * We are now going to push this context, so add it to the committing
1239 : * list before we do anything else. This ensures that anyone waiting on
1240 : * this push can easily detect the difference between a "push in
1241 : * progress" and "CIL is empty, nothing to do".
1242 : *
1243 : * IOWs, a wait loop can now check for:
1244 : * the current sequence not being found on the committing list;
1245 : * an empty CIL; and
1246 : * an unchanged sequence number
1247 : * to detect a push that had nothing to do and therefore does not need
1248 : * waiting on. If the CIL is not empty, we get put on the committing
1249 : * list before emptying the CIL and bumping the sequence number. Hence
1250 : * an empty CIL and an unchanged sequence number means we jumped out
1251 : * above after doing nothing.
1252 : *
1253 : * Hence the waiter will either find the commit sequence on the
1254 : * committing list or the sequence number will be unchanged and the CIL
1255 : * still dirty. In that latter case, the push has not yet started, and
1256 : * so the waiter will have to continue trying to check the CIL
1257 : * committing list until it is found. In extreme cases of delay, the
1258 : * sequence may fully commit between the attempts the wait makes to wait
1259 : * on the commit sequence.
1260 : */
1261 3762401 : list_add(&ctx->committing, &cil->xc_committing);
1262 3762402 : spin_unlock(&cil->xc_push_lock);
1263 :
1264 3762402 : xlog_cil_build_lv_chain(ctx, &whiteouts, &num_iovecs, &num_bytes);
1265 :
1266 : /*
1267 : * Switch the contexts so we can drop the context lock and move out
1268 : * of a shared context. We can't just go straight to the commit record,
1269 : * though - we need to synchronise with previous and future commits so
1270 : * that the commit records are correctly ordered in the log to ensure
1271 : * that we process items during log IO completion in the correct order.
1272 : *
1273 : * For example, if we get an EFI in one checkpoint and the EFD in the
1274 : * next (e.g. due to log forces), we do not want the checkpoint with
1275 : * the EFD to be committed before the checkpoint with the EFI. Hence
1276 : * we must strictly order the commit records of the checkpoints so
1277 : * that: a) the checkpoint callbacks are attached to the iclogs in the
1278 : * correct order; and b) the checkpoints are replayed in correct order
1279 : * in log recovery.
1280 : *
1281 : * Hence we need to add this context to the committing context list so
1282 : * that higher sequences will wait for us to write out a commit record
1283 : * before they do.
1284 : *
1285 : * xfs_log_force_seq requires us to mirror the new sequence into the cil
1286 : * structure atomically with the addition of this sequence to the
1287 : * committing list. This also ensures that we can do unlocked checks
1288 : * against the current sequence in log forces without risking
1289 : * deferencing a freed context pointer.
1290 : */
1291 3762401 : spin_lock(&cil->xc_push_lock);
1292 3762400 : xlog_cil_ctx_switch(cil, new_ctx);
1293 3762398 : spin_unlock(&cil->xc_push_lock);
1294 3762399 : up_write(&cil->xc_ctx_lock);
1295 :
1296 : /*
1297 : * Sort the log vector chain before we add the transaction headers.
1298 : * This ensures we always have the transaction headers at the start
1299 : * of the chain.
1300 : */
1301 3762401 : list_sort(NULL, &ctx->lv_chain, xlog_cil_order_cmp);
1302 :
1303 : /*
1304 : * Build a checkpoint transaction header and write it to the log to
1305 : * begin the transaction. We need to account for the space used by the
1306 : * transaction header here as it is not accounted for in xlog_write().
1307 : * Add the lvhdr to the head of the lv chain we pass to xlog_write() so
1308 : * it gets written into the iclog first.
1309 : */
1310 3762401 : xlog_cil_build_trans_hdr(ctx, &thdr, &lvhdr, num_iovecs);
1311 3762401 : num_bytes += lvhdr.lv_bytes;
1312 3762401 : list_add(&lvhdr.lv_list, &ctx->lv_chain);
1313 :
1314 : /*
1315 : * Take the lvhdr back off the lv_chain immediately after calling
1316 : * xlog_cil_write_chain() as it should not be passed to log IO
1317 : * completion.
1318 : */
1319 3762400 : error = xlog_cil_write_chain(ctx, num_bytes);
1320 3762399 : list_del(&lvhdr.lv_list);
1321 3762402 : if (error)
1322 7162 : goto out_abort_free_ticket;
1323 :
1324 3755240 : error = xlog_cil_write_commit_record(ctx);
1325 3755239 : if (error)
1326 0 : goto out_abort_free_ticket;
1327 :
1328 : /*
1329 : * Grab the ticket from the ctx so we can ungrant it after releasing the
1330 : * commit_iclog. The ctx may be freed by the time we return from
1331 : * releasing the commit_iclog (i.e. checkpoint has been completed and
1332 : * callback run) so we can't reference the ctx after the call to
1333 : * xlog_state_release_iclog().
1334 : */
1335 3755239 : ticket = ctx->ticket;
1336 :
1337 : /*
1338 : * If the checkpoint spans multiple iclogs, wait for all previous iclogs
1339 : * to complete before we submit the commit_iclog. We can't use state
1340 : * checks for this - ACTIVE can be either a past completed iclog or a
1341 : * future iclog being filled, while WANT_SYNC through SYNC_DONE can be a
1342 : * past or future iclog awaiting IO or ordered IO completion to be run.
1343 : * In the latter case, if it's a future iclog and we wait on it, the we
1344 : * will hang because it won't get processed through to ic_force_wait
1345 : * wakeup until this commit_iclog is written to disk. Hence we use the
1346 : * iclog header lsn and compare it to the commit lsn to determine if we
1347 : * need to wait on iclogs or not.
1348 : */
1349 3755239 : spin_lock(&log->l_icloglock);
1350 3755240 : if (ctx->start_lsn != ctx->commit_lsn) {
1351 2412329 : xfs_lsn_t plsn;
1352 :
1353 2412329 : plsn = be64_to_cpu(ctx->commit_iclog->ic_prev->ic_header.h_lsn);
1354 2412329 : if (plsn && XFS_LSN_CMP(plsn, ctx->commit_lsn) < 0) {
1355 : /*
1356 : * Waiting on ic_force_wait orders the completion of
1357 : * iclogs older than ic_prev. Hence we only need to wait
1358 : * on the most recent older iclog here.
1359 : */
1360 2356010 : xlog_wait_on_iclog(ctx->commit_iclog->ic_prev);
1361 2356008 : spin_lock(&log->l_icloglock);
1362 : }
1363 :
1364 : /*
1365 : * We need to issue a pre-flush so that the ordering for this
1366 : * checkpoint is correctly preserved down to stable storage.
1367 : */
1368 2412329 : ctx->commit_iclog->ic_flags |= XLOG_ICL_NEED_FLUSH;
1369 : }
1370 :
1371 : /*
1372 : * The commit iclog must be written to stable storage to guarantee
1373 : * journal IO vs metadata writeback IO is correctly ordered on stable
1374 : * storage.
1375 : *
1376 : * If the push caller needs the commit to be immediately stable and the
1377 : * commit_iclog is not yet marked as XLOG_STATE_WANT_SYNC to indicate it
1378 : * will be written when released, switch it's state to WANT_SYNC right
1379 : * now.
1380 : */
1381 3755240 : ctx->commit_iclog->ic_flags |= XLOG_ICL_NEED_FUA;
1382 3755240 : if (push_commit_stable &&
1383 132711 : ctx->commit_iclog->ic_state == XLOG_STATE_ACTIVE)
1384 128732 : xlog_state_switch_iclogs(log, ctx->commit_iclog, 0);
1385 3755240 : ticket = ctx->ticket;
1386 3755240 : xlog_state_release_iclog(log, ctx->commit_iclog, ticket);
1387 :
1388 : /* Not safe to reference ctx now! */
1389 :
1390 3755240 : spin_unlock(&log->l_icloglock);
1391 3755240 : xlog_cil_cleanup_whiteouts(&whiteouts);
1392 3755237 : xfs_log_ticket_ungrant(log, ticket);
1393 7517638 : return;
1394 :
1395 0 : out_skip:
1396 0 : up_write(&cil->xc_ctx_lock);
1397 0 : xfs_log_ticket_put(new_ctx->ticket);
1398 0 : kmem_free(new_ctx);
1399 : return;
1400 :
1401 7162 : out_abort_free_ticket:
1402 14324 : ASSERT(xlog_is_shutdown(log));
1403 7162 : xlog_cil_cleanup_whiteouts(&whiteouts);
1404 7162 : if (!ctx->commit_iclog) {
1405 7162 : xfs_log_ticket_ungrant(log, ctx->ticket);
1406 7162 : xlog_cil_committed(ctx);
1407 7162 : return;
1408 : }
1409 0 : spin_lock(&log->l_icloglock);
1410 0 : ticket = ctx->ticket;
1411 0 : xlog_state_release_iclog(log, ctx->commit_iclog, ticket);
1412 : /* Not safe to reference ctx now! */
1413 0 : spin_unlock(&log->l_icloglock);
1414 0 : xfs_log_ticket_ungrant(log, ticket);
1415 : }
1416 :
1417 : /*
1418 : * We need to push CIL every so often so we don't cache more than we can fit in
1419 : * the log. The limit really is that a checkpoint can't be more than half the
1420 : * log (the current checkpoint is not allowed to overwrite the previous
1421 : * checkpoint), but commit latency and memory usage limit this to a smaller
1422 : * size.
1423 : */
1424 : static void
1425 1774722125 : xlog_cil_push_background(
1426 : struct xlog *log) __releases(cil->xc_ctx_lock)
1427 : {
1428 1774722125 : struct xfs_cil *cil = log->l_cilp;
1429 1774722125 : int space_used = atomic_read(&cil->xc_ctx->space_used);
1430 :
1431 : /*
1432 : * The cil won't be empty because we are called while holding the
1433 : * context lock so whatever we added to the CIL will still be there.
1434 : */
1435 1774722125 : ASSERT(!test_bit(XLOG_CIL_EMPTY, &cil->xc_flags));
1436 :
1437 : /*
1438 : * We are done if:
1439 : * - we haven't used up all the space available yet; or
1440 : * - we've already queued up a push; and
1441 : * - we're not over the hard limit; and
1442 : * - nothing has been over the hard limit.
1443 : *
1444 : * If so, we don't need to take the push lock as there's nothing to do.
1445 : */
1446 1774722131 : if (space_used < XLOG_CIL_SPACE_LIMIT(log) ||
1447 4978 : (cil->xc_push_seq == cil->xc_current_sequence &&
1448 4061 : space_used < XLOG_CIL_BLOCKING_SPACE_LIMIT(log) &&
1449 : !waitqueue_active(&cil->xc_push_wait))) {
1450 1774721208 : up_read(&cil->xc_ctx_lock);
1451 1774721208 : return;
1452 : }
1453 :
1454 917 : spin_lock(&cil->xc_push_lock);
1455 917 : if (cil->xc_push_seq < cil->xc_current_sequence) {
1456 917 : cil->xc_push_seq = cil->xc_current_sequence;
1457 917 : queue_work(cil->xc_push_wq, &cil->xc_ctx->push_work);
1458 : }
1459 :
1460 : /*
1461 : * Drop the context lock now, we can't hold that if we need to sleep
1462 : * because we are over the blocking threshold. The push_lock is still
1463 : * held, so blocking threshold sleep/wakeup is still correctly
1464 : * serialised here.
1465 : */
1466 917 : up_read(&cil->xc_ctx_lock);
1467 :
1468 : /*
1469 : * If we are well over the space limit, throttle the work that is being
1470 : * done until the push work on this context has begun. Enforce the hard
1471 : * throttle on all transaction commits once it has been activated, even
1472 : * if the committing transactions have resulted in the space usage
1473 : * dipping back down under the hard limit.
1474 : *
1475 : * The ctx->xc_push_lock provides the serialisation necessary for safely
1476 : * calling xlog_cil_over_hard_limit() in this context.
1477 : */
1478 917 : if (xlog_cil_over_hard_limit(log, space_used)) {
1479 0 : trace_xfs_log_cil_wait(log, cil->xc_ctx->ticket);
1480 0 : ASSERT(space_used < log->l_logsize);
1481 0 : xlog_wait(&cil->xc_push_wait, &cil->xc_push_lock);
1482 0 : return;
1483 : }
1484 :
1485 917 : spin_unlock(&cil->xc_push_lock);
1486 :
1487 : }
1488 :
1489 : /*
1490 : * xlog_cil_push_now() is used to trigger an immediate CIL push to the sequence
1491 : * number that is passed. When it returns, the work will be queued for
1492 : * @push_seq, but it won't be completed.
1493 : *
1494 : * If the caller is performing a synchronous force, we will flush the workqueue
1495 : * to get previously queued work moving to minimise the wait time they will
1496 : * undergo waiting for all outstanding pushes to complete. The caller is
1497 : * expected to do the required waiting for push_seq to complete.
1498 : *
1499 : * If the caller is performing an async push, we need to ensure that the
1500 : * checkpoint is fully flushed out of the iclogs when we finish the push. If we
1501 : * don't do this, then the commit record may remain sitting in memory in an
1502 : * ACTIVE iclog. This then requires another full log force to push to disk,
1503 : * which defeats the purpose of having an async, non-blocking CIL force
1504 : * mechanism. Hence in this case we need to pass a flag to the push work to
1505 : * indicate it needs to flush the commit record itself.
1506 : */
1507 : static void
1508 8669510 : xlog_cil_push_now(
1509 : struct xlog *log,
1510 : xfs_lsn_t push_seq,
1511 : bool async)
1512 : {
1513 8669510 : struct xfs_cil *cil = log->l_cilp;
1514 :
1515 8669510 : if (!cil)
1516 : return;
1517 :
1518 8669510 : ASSERT(push_seq && push_seq <= cil->xc_current_sequence);
1519 :
1520 : /* start on any pending background push to minimise wait time on it */
1521 8669510 : if (!async)
1522 8509555 : flush_workqueue(cil->xc_push_wq);
1523 :
1524 8669491 : spin_lock(&cil->xc_push_lock);
1525 :
1526 : /*
1527 : * If this is an async flush request, we always need to set the
1528 : * xc_push_commit_stable flag even if something else has already queued
1529 : * a push. The flush caller is asking for the CIL to be on stable
1530 : * storage when the next push completes, so regardless of who has queued
1531 : * the push, the flush requires stable semantics from it.
1532 : */
1533 8669854 : cil->xc_push_commit_stable = async;
1534 :
1535 : /*
1536 : * If the CIL is empty or we've already pushed the sequence then
1537 : * there's no more work that we need to do.
1538 : */
1539 8669854 : if (test_bit(XLOG_CIL_EMPTY, &cil->xc_flags) ||
1540 5372295 : push_seq <= cil->xc_push_seq) {
1541 4908371 : spin_unlock(&cil->xc_push_lock);
1542 4908371 : return;
1543 : }
1544 :
1545 3761483 : cil->xc_push_seq = push_seq;
1546 3761483 : queue_work(cil->xc_push_wq, &cil->xc_ctx->push_work);
1547 3761485 : spin_unlock(&cil->xc_push_lock);
1548 : }
1549 :
1550 : bool
1551 271076 : xlog_cil_empty(
1552 : struct xlog *log)
1553 : {
1554 271076 : struct xfs_cil *cil = log->l_cilp;
1555 271076 : bool empty = false;
1556 :
1557 271076 : spin_lock(&cil->xc_push_lock);
1558 542146 : if (test_bit(XLOG_CIL_EMPTY, &cil->xc_flags))
1559 266257 : empty = true;
1560 271073 : spin_unlock(&cil->xc_push_lock);
1561 271068 : return empty;
1562 : }
1563 :
1564 : /*
1565 : * If there are intent done items in this transaction and the related intent was
1566 : * committed in the current (same) CIL checkpoint, we don't need to write either
1567 : * the intent or intent done item to the journal as the change will be
1568 : * journalled atomically within this checkpoint. As we cannot remove items from
1569 : * the CIL here, mark the related intent with a whiteout so that the CIL push
1570 : * can remove it rather than writing it to the journal. Then remove the intent
1571 : * done item from the current transaction and release it so it doesn't get put
1572 : * into the CIL at all.
1573 : */
1574 : static uint32_t
1575 1172112891 : xlog_cil_process_intents(
1576 : struct xfs_cil *cil,
1577 : struct xfs_trans *tp)
1578 : {
1579 1172112891 : struct xfs_log_item *lip, *ilip, *next;
1580 1172112891 : uint32_t len = 0;
1581 :
1582 8674128365 : list_for_each_entry_safe(lip, next, &tp->t_items, li_trans) {
1583 7502014906 : if (!(lip->li_ops->flags & XFS_ITEM_INTENT_DONE))
1584 6686455956 : continue;
1585 :
1586 815558950 : ilip = lip->li_ops->iop_intent(lip);
1587 815559315 : if (!ilip || !xlog_item_in_current_chkpt(cil, ilip))
1588 784838 : continue;
1589 814774618 : set_bit(XFS_LI_WHITEOUT, &ilip->li_flags);
1590 814774666 : trace_xfs_cil_whiteout_mark(ilip);
1591 814774856 : len += ilip->li_lv->lv_bytes;
1592 814774856 : kmem_free(ilip->li_lv);
1593 814775008 : ilip->li_lv = NULL;
1594 :
1595 814775008 : xfs_trans_del_item(lip);
1596 814775074 : lip->li_ops->iop_release(lip);
1597 : }
1598 1172113459 : return len;
1599 : }
1600 :
1601 : /*
1602 : * Commit a transaction with the given vector to the Committed Item List.
1603 : *
1604 : * To do this, we need to format the item, pin it in memory if required and
1605 : * account for the space used by the transaction. Once we have done that we
1606 : * need to release the unused reservation for the transaction, attach the
1607 : * transaction to the checkpoint context so we carry the busy extents through
1608 : * to checkpoint completion, and then unlock all the items in the transaction.
1609 : *
1610 : * Called with the context lock already held in read mode to lock out
1611 : * background commit, returns without it held once background commits are
1612 : * allowed again.
1613 : */
1614 : void
1615 1774881733 : xlog_cil_commit(
1616 : struct xlog *log,
1617 : struct xfs_trans *tp,
1618 : xfs_csn_t *commit_seq,
1619 : bool regrant)
1620 : {
1621 1774881733 : struct xfs_cil *cil = log->l_cilp;
1622 1774881733 : struct xfs_log_item *lip, *next;
1623 1774881733 : uint32_t released_space = 0;
1624 :
1625 : /*
1626 : * Do all necessary memory allocation before we lock the CIL.
1627 : * This ensures the allocation does not deadlock with a CIL
1628 : * push in memory reclaim (e.g. from kswapd).
1629 : */
1630 1774881733 : xlog_cil_alloc_shadow_bufs(log, tp);
1631 :
1632 : /* lock out background commit */
1633 1774814130 : down_read(&cil->xc_ctx_lock);
1634 :
1635 1774922254 : if (tp->t_flags & XFS_TRANS_HAS_INTENT_DONE)
1636 1172113140 : released_space = xlog_cil_process_intents(cil, tp);
1637 :
1638 1774922380 : xlog_cil_insert_items(log, tp, released_space);
1639 :
1640 2737053133 : if (regrant && !xlog_is_shutdown(log))
1641 962016067 : xfs_log_ticket_regrant(log, tp->t_ticket);
1642 : else
1643 813022112 : xfs_log_ticket_ungrant(log, tp->t_ticket);
1644 1775069050 : tp->t_ticket = NULL;
1645 1775069050 : xfs_trans_unreserve_and_mod_sb(tp);
1646 :
1647 : /*
1648 : * Once all the items of the transaction have been copied to the CIL,
1649 : * the items can be unlocked and possibly freed.
1650 : *
1651 : * This needs to be done before we drop the CIL context lock because we
1652 : * have to update state in the log items and unlock them before they go
1653 : * to disk. If we don't, then the CIL checkpoint can race with us and
1654 : * we can run checkpoint completion before we've updated and unlocked
1655 : * the log items. This affects (at least) processing of stale buffers,
1656 : * inodes and EFIs.
1657 : */
1658 1775042496 : trace_xfs_trans_commit_items(tp, _RET_IP_);
1659 12204495279 : list_for_each_entry_safe(lip, next, &tp->t_items, li_trans) {
1660 10429619770 : xfs_trans_del_item(lip);
1661 10429681257 : if (lip->li_ops->iop_committing)
1662 9610021813 : lip->li_ops->iop_committing(lip, cil->xc_ctx->sequence);
1663 : }
1664 1774875509 : if (commit_seq)
1665 1774875509 : *commit_seq = cil->xc_ctx->sequence;
1666 :
1667 : /* xlog_cil_push_background() releases cil->xc_ctx_lock */
1668 1774875509 : xlog_cil_push_background(log);
1669 1774875113 : }
1670 :
1671 : /*
1672 : * Flush the CIL to stable storage but don't wait for it to complete. This
1673 : * requires the CIL push to ensure the commit record for the push hits the disk,
1674 : * but otherwise is no different to a push done from a log force.
1675 : */
1676 : void
1677 159948 : xlog_cil_flush(
1678 : struct xlog *log)
1679 : {
1680 159948 : xfs_csn_t seq = log->l_cilp->xc_current_sequence;
1681 :
1682 159948 : trace_xfs_log_force(log->l_mp, seq, _RET_IP_);
1683 159948 : xlog_cil_push_now(log, seq, true);
1684 :
1685 : /*
1686 : * If the CIL is empty, make sure that any previous checkpoint that may
1687 : * still be in an active iclog is pushed to stable storage.
1688 : */
1689 319896 : if (test_bit(XLOG_CIL_EMPTY, &log->l_cilp->xc_flags))
1690 24258 : xfs_log_force(log->l_mp, 0);
1691 159948 : }
1692 :
1693 : /*
1694 : * Conditionally push the CIL based on the sequence passed in.
1695 : *
1696 : * We only need to push if we haven't already pushed the sequence number given.
1697 : * Hence the only time we will trigger a push here is if the push sequence is
1698 : * the same as the current context.
1699 : *
1700 : * We return the current commit lsn to allow the callers to determine if a
1701 : * iclog flush is necessary following this call.
1702 : */
1703 : xfs_lsn_t
1704 5846339 : xlog_cil_force_seq(
1705 : struct xlog *log,
1706 : xfs_csn_t sequence)
1707 : {
1708 5846339 : struct xfs_cil *cil = log->l_cilp;
1709 5846339 : struct xfs_cil_ctx *ctx;
1710 5846339 : xfs_lsn_t commit_lsn = NULLCOMMITLSN;
1711 :
1712 5846339 : ASSERT(sequence <= cil->xc_current_sequence);
1713 :
1714 5846339 : if (!sequence)
1715 0 : sequence = cil->xc_current_sequence;
1716 5846339 : trace_xfs_log_force(log->l_mp, sequence, _RET_IP_);
1717 :
1718 : /*
1719 : * check to see if we need to force out the current context.
1720 : * xlog_cil_push() handles racing pushes for the same sequence,
1721 : * so no need to deal with it here.
1722 : */
1723 : restart:
1724 8509649 : xlog_cil_push_now(log, sequence, false);
1725 :
1726 : /*
1727 : * See if we can find a previous sequence still committing.
1728 : * We need to wait for all previous sequence commits to complete
1729 : * before allowing the force of push_seq to go ahead. Hence block
1730 : * on commits for those as well.
1731 : */
1732 8509849 : spin_lock(&cil->xc_push_lock);
1733 13622089 : list_for_each_entry(ctx, &cil->xc_committing, committing) {
1734 : /*
1735 : * Avoid getting stuck in this loop because we were woken by the
1736 : * shutdown, but then went back to sleep once already in the
1737 : * shutdown state.
1738 : */
1739 12003930 : if (xlog_is_shutdown(log))
1740 330 : goto out_shutdown;
1741 6001635 : if (ctx->sequence > sequence)
1742 379226 : continue;
1743 5622409 : if (!ctx->commit_lsn) {
1744 : /*
1745 : * It is still being pushed! Wait for the push to
1746 : * complete, then start again from the beginning.
1747 : */
1748 889454 : XFS_STATS_INC(log->l_mp, xs_log_force_sleep);
1749 889454 : xlog_wait(&cil->xc_commit_wait, &cil->xc_push_lock);
1750 889442 : goto restart;
1751 : }
1752 4732955 : if (ctx->sequence != sequence)
1753 604866 : continue;
1754 : /* found it! */
1755 : commit_lsn = ctx->commit_lsn;
1756 : }
1757 :
1758 : /*
1759 : * The call to xlog_cil_push_now() executes the push in the background.
1760 : * Hence by the time we have got here it our sequence may not have been
1761 : * pushed yet. This is true if the current sequence still matches the
1762 : * push sequence after the above wait loop and the CIL still contains
1763 : * dirty objects. This is guaranteed by the push code first adding the
1764 : * context to the committing list before emptying the CIL.
1765 : *
1766 : * Hence if we don't find the context in the committing list and the
1767 : * current sequence number is unchanged then the CIL contents are
1768 : * significant. If the CIL is empty, if means there was nothing to push
1769 : * and that means there is nothing to wait for. If the CIL is not empty,
1770 : * it means we haven't yet started the push, because if it had started
1771 : * we would have found the context on the committing list.
1772 : */
1773 10971486 : if (sequence == cil->xc_current_sequence &&
1774 3351362 : !test_bit(XLOG_CIL_EMPTY, &cil->xc_flags)) {
1775 1773779 : spin_unlock(&cil->xc_push_lock);
1776 1773776 : goto restart;
1777 : }
1778 :
1779 5846345 : spin_unlock(&cil->xc_push_lock);
1780 5846345 : return commit_lsn;
1781 :
1782 : /*
1783 : * We detected a shutdown in progress. We need to trigger the log force
1784 : * to pass through it's iclog state machine error handling, even though
1785 : * we are already in a shutdown state. Hence we can't return
1786 : * NULLCOMMITLSN here as that has special meaning to log forces (i.e.
1787 : * LSN is already stable), so we return a zero LSN instead.
1788 : */
1789 : out_shutdown:
1790 330 : spin_unlock(&cil->xc_push_lock);
1791 330 : return 0;
1792 : }
1793 :
1794 : /*
1795 : * Move dead percpu state to the relevant CIL context structures.
1796 : *
1797 : * We have to lock the CIL context here to ensure that nothing is modifying
1798 : * the percpu state, either addition or removal. Both of these are done under
1799 : * the CIL context lock, so grabbing that exclusively here will ensure we can
1800 : * safely drain the cilpcp for the CPU that is dying.
1801 : */
1802 : void
1803 9 : xlog_cil_pcp_dead(
1804 : struct xlog *log,
1805 : unsigned int cpu)
1806 : {
1807 9 : struct xfs_cil *cil = log->l_cilp;
1808 9 : struct xlog_cil_pcp *cilpcp = per_cpu_ptr(cil->xc_pcp, cpu);
1809 9 : struct xfs_cil_ctx *ctx;
1810 :
1811 9 : down_write(&cil->xc_ctx_lock);
1812 9 : ctx = cil->xc_ctx;
1813 9 : if (ctx->ticket)
1814 9 : ctx->ticket->t_curr_res += cilpcp->space_reserved;
1815 9 : cilpcp->space_reserved = 0;
1816 :
1817 9 : if (!list_empty(&cilpcp->log_items))
1818 1 : list_splice_init(&cilpcp->log_items, &ctx->log_items);
1819 9 : if (!list_empty(&cilpcp->busy_extents))
1820 0 : list_splice_init(&cilpcp->busy_extents, &ctx->busy_extents);
1821 9 : atomic_add(cilpcp->space_used, &ctx->space_used);
1822 9 : cilpcp->space_used = 0;
1823 9 : up_write(&cil->xc_ctx_lock);
1824 9 : }
1825 :
1826 : /*
1827 : * Perform initial CIL structure initialisation.
1828 : */
1829 : int
1830 24333 : xlog_cil_init(
1831 : struct xlog *log)
1832 : {
1833 24333 : struct xfs_cil *cil;
1834 24333 : struct xfs_cil_ctx *ctx;
1835 24333 : struct xlog_cil_pcp *cilpcp;
1836 24333 : int cpu;
1837 :
1838 24333 : cil = kmem_zalloc(sizeof(*cil), KM_MAYFAIL);
1839 24333 : if (!cil)
1840 : return -ENOMEM;
1841 : /*
1842 : * Limit the CIL pipeline depth to 4 concurrent works to bound the
1843 : * concurrency the log spinlocks will be exposed to.
1844 : */
1845 48666 : cil->xc_push_wq = alloc_workqueue("xfs-cil/%s",
1846 : XFS_WQFLAGS(WQ_FREEZABLE | WQ_MEM_RECLAIM | WQ_UNBOUND),
1847 24333 : 4, log->l_mp->m_super->s_id);
1848 24333 : if (!cil->xc_push_wq)
1849 0 : goto out_destroy_cil;
1850 :
1851 24333 : cil->xc_log = log;
1852 24333 : cil->xc_pcp = alloc_percpu(struct xlog_cil_pcp);
1853 24333 : if (!cil->xc_pcp)
1854 0 : goto out_destroy_wq;
1855 :
1856 72999 : for_each_possible_cpu(cpu) {
1857 48666 : cilpcp = per_cpu_ptr(cil->xc_pcp, cpu);
1858 48666 : INIT_LIST_HEAD(&cilpcp->busy_extents);
1859 48666 : INIT_LIST_HEAD(&cilpcp->log_items);
1860 : }
1861 :
1862 24333 : INIT_LIST_HEAD(&cil->xc_committing);
1863 24333 : spin_lock_init(&cil->xc_push_lock);
1864 24333 : init_waitqueue_head(&cil->xc_push_wait);
1865 24333 : init_rwsem(&cil->xc_ctx_lock);
1866 24333 : init_waitqueue_head(&cil->xc_start_wait);
1867 24333 : init_waitqueue_head(&cil->xc_commit_wait);
1868 24333 : log->l_cilp = cil;
1869 :
1870 24333 : ctx = xlog_cil_ctx_alloc();
1871 24333 : xlog_cil_ctx_switch(cil, ctx);
1872 24333 : return 0;
1873 :
1874 : out_destroy_wq:
1875 0 : destroy_workqueue(cil->xc_push_wq);
1876 0 : out_destroy_cil:
1877 0 : kmem_free(cil);
1878 0 : return -ENOMEM;
1879 : }
1880 :
1881 : void
1882 24339 : xlog_cil_destroy(
1883 : struct xlog *log)
1884 : {
1885 24339 : struct xfs_cil *cil = log->l_cilp;
1886 :
1887 24339 : if (cil->xc_ctx) {
1888 24339 : if (cil->xc_ctx->ticket)
1889 24331 : xfs_log_ticket_put(cil->xc_ctx->ticket);
1890 24339 : kmem_free(cil->xc_ctx);
1891 : }
1892 :
1893 24339 : ASSERT(test_bit(XLOG_CIL_EMPTY, &cil->xc_flags));
1894 24339 : free_percpu(cil->xc_pcp);
1895 24339 : destroy_workqueue(cil->xc_push_wq);
1896 24339 : kmem_free(cil);
1897 24339 : }
1898 :
|
