Line data    Source code

       1             : // SPDX-License-Identifier: GPL-2.0-or-later
       2             : /*
       3             :  * Copyright (C) 2022-2023 Oracle.  All Rights Reserved.
       4             :  * Author: Darrick J. Wong <djwong@kernel.org>
       5             :  */
       6             : #include "xfs.h"
       7             : #include "xfs_fs.h"
       8             : #include "xfs_shared.h"
       9             : #include "xfs_format.h"
      10             : #include "xfs_trans_resv.h"
      11             : #include "xfs_mount.h"
      12             : #include "xfs_ag.h"
      13             : #include "xfs_trace.h"
      14             : 
      15             : /*
      16             :  * Use a static key here to reduce the overhead of xfs_drain_rele.  If the
      17             :  * compiler supports jump labels, the static branch will be replaced by a nop
      18             :  * sled when there are no xfs_drain_wait callers.  Online fsck is currently
      19             :  * the only caller, so this is a reasonable tradeoff.
      20             :  *
      21             :  * Note: Patching the kernel code requires taking the cpu hotplug lock.  Other
      22             :  * parts of the kernel allocate memory with that lock held, which means that
      23             :  * XFS callers cannot hold any locks that might be used by memory reclaim or
      24             :  * writeback when calling the static_branch_{inc,dec} functions.
      25             :  */
      26             : static DEFINE_STATIC_KEY_FALSE(xfs_drain_waiter_gate);
      27             : 
      28             : void
      29       33656 : xfs_drain_wait_disable(void)
      30             : {
      31       33656 :         static_branch_dec(&xfs_drain_waiter_gate);
      32       33656 : }
      33             : 
      34             : void
      35       33656 : xfs_drain_wait_enable(void)
      36             : {
      37       33656 :         static_branch_inc(&xfs_drain_waiter_gate);
      38       33656 : }
      39             : 
      40             : void
      41      133388 : xfs_defer_drain_init(
      42             :         struct xfs_defer_drain  *dr)
      43             : {
      44      133388 :         atomic_set(&dr->dr_count, 0);
      45      133388 :         init_waitqueue_head(&dr->dr_waiters);
      46      133388 : }
      47             : 
      48             : void
      49      133416 : xfs_defer_drain_free(struct xfs_defer_drain     *dr)
      50             : {
      51      133416 :         ASSERT(atomic_read(&dr->dr_count) == 0);
      52      133416 : }
      53             : 
      54             : /* Increase the pending intent count. */
      55   452860487 : static inline void xfs_defer_drain_grab(struct xfs_defer_drain *dr)
      56             : {
      57   452860487 :         atomic_inc(&dr->dr_count);
      58   452848648 : }
      59             : 
      60             : static inline bool has_waiters(struct wait_queue_head *wq_head)
      61             : {
      62             :         /*
      63             :          * This memory barrier is paired with the one in set_current_state on
      64             :          * the waiting side.
      65             :          */
      66      349312 :         smp_mb__after_atomic();
      67      349311 :         return waitqueue_active(wq_head);
      68             : }
      69             : 
      70             : /* Decrease the pending intent count, and wake any waiters, if appropriate. */
      71   452811491 : static inline void xfs_defer_drain_rele(struct xfs_defer_drain *dr)
      72             : {
      73   905605005 :         if (atomic_dec_and_test(&dr->dr_count) &&
      74   167271297 :             static_branch_unlikely(&xfs_drain_waiter_gate) &&
      75             :             has_waiters(&dr->dr_waiters))
      76       12950 :                 wake_up(&dr->dr_waiters);
      77   452793654 : }
      78             : 
      79             : /* Are there intents pending? */
      80             : static inline bool xfs_defer_drain_busy(struct xfs_defer_drain *dr)
      81             : {
      82     2481692 :         return atomic_read(&dr->dr_count) > 0;
      83             : }
      84             : 
      85             : /*
      86             :  * Wait for the pending intent count for a drain to hit zero.
      87             :  *
      88             :  * Callers must not hold any locks that would prevent intents from being
      89             :  * finished.
      90             :  */
      91       12317 : static inline int xfs_defer_drain_wait(struct xfs_defer_drain *dr)
      92             : {
      93       25265 :         return wait_event_killable(dr->dr_waiters, !xfs_defer_drain_busy(dr));
      94             : }
      95             : 
      96             : /*
      97             :  * Get a passive reference to an AG and declare an intent to update its
      98             :  * metadata.
      99             :  */
     100             : struct xfs_perag *
     101   452768777 : xfs_perag_intent_get(
     102             :         struct xfs_mount        *mp,
     103             :         xfs_agnumber_t          agno)
     104             : {
     105   452768777 :         struct xfs_perag        *pag;
     106             : 
     107   452768777 :         pag = xfs_perag_get(mp, agno);
     108   452774915 :         if (!pag)
     109             :                 return NULL;
     110             : 
     111   452774915 :         xfs_perag_intent_hold(pag);
     112   452774915 :         return pag;
     113             : }
     114             : 
     115             : /*
     116             :  * Release our intent to update this AG's metadata, and then release our
     117             :  * passive ref to the AG.
     118             :  */
     119             : void
     120   452729559 : xfs_perag_intent_put(
     121             :         struct xfs_perag        *pag)
     122             : {
     123   452729559 :         xfs_perag_intent_rele(pag);
     124   452713780 :         xfs_perag_put(pag);
     125   452714608 : }
     126             : 
     127             : /*
     128             :  * Declare an intent to update AG metadata.  Other threads that need exclusive
     129             :  * access can decide to back off if they see declared intentions.
     130             :  */
     131             : void
     132   452859573 : xfs_perag_intent_hold(
     133             :         struct xfs_perag        *pag)
     134             : {
     135   452859573 :         trace_xfs_perag_intent_hold(pag, __return_address);
     136   452861694 :         xfs_defer_drain_grab(&pag->pag_intents_drain);
     137   452861082 : }
     138             : 
     139             : /* Release our intent to update this AG's metadata. */
     140             : void
     141   452806378 : xfs_perag_intent_rele(
     142             :         struct xfs_perag        *pag)
     143             : {
     144   452806378 :         trace_xfs_perag_intent_rele(pag, __return_address);
     145   452810255 :         xfs_defer_drain_rele(&pag->pag_intents_drain);
     146   452795101 : }
     147             : 
     148             : /*
     149             :  * Wait for the intent update count for this AG to hit zero.
     150             :  * Callers must not hold any AG header buffers.
     151             :  */
     152             : int
     153       12317 : xfs_perag_intent_drain(
     154             :         struct xfs_perag        *pag)
     155             : {
     156       12317 :         trace_xfs_perag_wait_intents(pag, __return_address);
     157       12317 :         return xfs_defer_drain_wait(&pag->pag_intents_drain);
     158             : }
     159             : 
     160             : /* Has anyone declared an intent to update this AG? */
     161             : bool
     162     2444635 : xfs_perag_intent_busy(
     163             :         struct xfs_perag        *pag)
     164             : {
     165     2444635 :         return xfs_defer_drain_busy(&pag->pag_intents_drain);
     166             : }