Line data    Source code

       1             : // SPDX-License-Identifier: GPL-2.0-or-later
       2             : /*
       3             :  * Copyright (C) 2022-2023 Oracle.  All Rights Reserved.
       4             :  * Author: Darrick J. Wong <djwong@kernel.org>
       5             :  */
       6             : #include "xfs.h"
       7             : #include "xfs_fs.h"
       8             : #include "xfs_shared.h"
       9             : #include "xfs_format.h"
      10             : #include "xfs_trans_resv.h"
      11             : #include "xfs_mount.h"
      12             : #include "xfs_ag.h"
      13             : #include "xfs_trace.h"
      14             : 
      15             : /*
      16             :  * Use a static key here to reduce the overhead of xfs_drain_rele.  If the
      17             :  * compiler supports jump labels, the static branch will be replaced by a nop
      18             :  * sled when there are no xfs_drain_wait callers.  Online fsck is currently
      19             :  * the only caller, so this is a reasonable tradeoff.
      20             :  *
      21             :  * Note: Patching the kernel code requires taking the cpu hotplug lock.  Other
      22             :  * parts of the kernel allocate memory with that lock held, which means that
      23             :  * XFS callers cannot hold any locks that might be used by memory reclaim or
      24             :  * writeback when calling the static_branch_{inc,dec} functions.
      25             :  */
      26             : static DEFINE_STATIC_KEY_FALSE(xfs_drain_waiter_gate);
      27             : 
      28             : void
      29       49232 : xfs_drain_wait_disable(void)
      30             : {
      31       49232 :         static_branch_dec(&xfs_drain_waiter_gate);
      32       49232 : }
      33             : 
      34             : void
      35       49232 : xfs_drain_wait_enable(void)
      36             : {
      37       49232 :         static_branch_inc(&xfs_drain_waiter_gate);
      38       49232 : }
      39             : 
      40             : void
      41      126902 : xfs_defer_drain_init(
      42             :         struct xfs_defer_drain  *dr)
      43             : {
      44      126902 :         atomic_set(&dr->dr_count, 0);
      45      126902 :         init_waitqueue_head(&dr->dr_waiters);
      46      126902 : }
      47             : 
      48             : void
      49      126914 : xfs_defer_drain_free(struct xfs_defer_drain     *dr)
      50             : {
      51      126914 :         ASSERT(atomic_read(&dr->dr_count) == 0);
      52      126914 : }
      53             : 
      54             : /* Increase the pending intent count. */
      55   404964353 : static inline void xfs_defer_drain_grab(struct xfs_defer_drain *dr)
      56             : {
      57   404964353 :         atomic_inc(&dr->dr_count);
      58   404964479 : }
      59             : 
      60             : static inline bool has_waiters(struct wait_queue_head *wq_head)
      61             : {
      62             :         /*
      63             :          * This memory barrier is paired with the one in set_current_state on
      64             :          * the waiting side.
      65             :          */
      66      183291 :         smp_mb__after_atomic();
      67      183290 :         return waitqueue_active(wq_head);
      68             : }
      69             : 
      70             : /* Decrease the pending intent count, and wake any waiters, if appropriate. */
      71   404915023 : static inline void xfs_defer_drain_rele(struct xfs_defer_drain *dr)
      72             : {
      73   809836354 :         if (atomic_dec_and_test(&dr->dr_count) &&
      74   144233729 :             static_branch_unlikely(&xfs_drain_waiter_gate) &&
      75             :             has_waiters(&dr->dr_waiters))
      76       31754 :                 wake_up(&dr->dr_waiters);
      77   404921221 : }
      78             : 
      79             : /* Are there intents pending? */
      80             : static inline bool xfs_defer_drain_busy(struct xfs_defer_drain *dr)
      81             : {
      82     8858776 :         return atomic_read(&dr->dr_count) > 0;
      83             : }
      84             : 
      85             : /*
      86             :  * Wait for the pending intent count for a drain to hit zero.
      87             :  *
      88             :  * Callers must not hold any locks that would prevent intents from being
      89             :  * finished.
      90             :  */
      91       29676 : static inline int xfs_defer_drain_wait(struct xfs_defer_drain *dr)
      92             : {
      93       61423 :         return wait_event_killable(dr->dr_waiters, !xfs_defer_drain_busy(dr));
      94             : }
      95             : 
      96             : /*
      97             :  * Get a passive reference to an AG and declare an intent to update its
      98             :  * metadata.
      99             :  */
     100             : struct xfs_perag *
     101   404960956 : xfs_perag_intent_get(
     102             :         struct xfs_mount        *mp,
     103             :         xfs_agnumber_t          agno)
     104             : {
     105   404960956 :         struct xfs_perag        *pag;
     106             : 
     107   404960956 :         pag = xfs_perag_get(mp, agno);
     108   404967498 :         if (!pag)
     109             :                 return NULL;
     110             : 
     111   404967498 :         xfs_perag_intent_hold(pag);
     112   404967498 :         return pag;
     113             : }
     114             : 
     115             : /*
     116             :  * Release our intent to update this AG's metadata, and then release our
     117             :  * passive ref to the AG.
     118             :  */
     119             : void
     120   404907735 : xfs_perag_intent_put(
     121             :         struct xfs_perag        *pag)
     122             : {
     123   404907735 :         xfs_perag_intent_rele(pag);
     124   404920070 :         xfs_perag_put(pag);
     125   404915954 : }
     126             : 
     127             : /*
     128             :  * Declare an intent to update AG metadata.  Other threads that need exclusive
     129             :  * access can decide to back off if they see declared intentions.
     130             :  */
     131             : void
     132   404961196 : xfs_perag_intent_hold(
     133             :         struct xfs_perag        *pag)
     134             : {
     135   404961196 :         trace_xfs_perag_intent_hold(pag, __return_address);
     136   404964414 :         xfs_defer_drain_grab(&pag->pag_intents_drain);
     137   404964237 : }
     138             : 
     139             : /* Release our intent to update this AG's metadata. */
     140             : void
     141   404911528 : xfs_perag_intent_rele(
     142             :         struct xfs_perag        *pag)
     143             : {
     144   404911528 :         trace_xfs_perag_intent_rele(pag, __return_address);
     145   404913200 :         xfs_defer_drain_rele(&pag->pag_intents_drain);
     146   404922383 : }
     147             : 
     148             : /*
     149             :  * Wait for the intent update count for this AG to hit zero.
     150             :  * Callers must not hold any AG header buffers.
     151             :  */
     152             : int
     153       29676 : xfs_perag_intent_drain(
     154             :         struct xfs_perag        *pag)
     155             : {
     156       29676 :         trace_xfs_perag_wait_intents(pag, __return_address);
     157       29676 :         return xfs_defer_drain_wait(&pag->pag_intents_drain);
     158             : }
     159             : 
     160             : /* Has anyone declared an intent to update this AG? */
     161             : bool
     162     8768045 : xfs_perag_intent_busy(
     163             :         struct xfs_perag        *pag)
     164             : {
     165     8768045 :         return xfs_defer_drain_busy(&pag->pag_intents_drain);
     166             : }