Line data    Source code

       1             : // SPDX-License-Identifier: GPL-2.0-or-later
       2             : /*
       3             :  * Copyright (C) 2022-2023 Oracle.  All Rights Reserved.
       4             :  * Author: Darrick J. Wong <djwong@kernel.org>
       5             :  */
       6             : #include "xfs.h"
       7             : #include "xfs_fs.h"
       8             : #include "xfs_shared.h"
       9             : #include "xfs_format.h"
      10             : #include "xfs_trans_resv.h"
      11             : #include "xfs_mount.h"
      12             : #include "xfs_ag.h"
      13             : #include "xfs_trace.h"
      14             : #include "xfs_rtgroup.h"
      15             : 
      16             : /*
      17             :  * Use a static key here to reduce the overhead of xfs_defer_drain_rele.  If the
      18             :  * compiler supports jump labels, the static branch will be replaced by a nop
      19             :  * sled when there are no xfs_defer_drain_wait callers.  Online fsck is currently
      20             :  * the only caller, so this is a reasonable tradeoff.
      21             :  *
      22             :  * Note: Patching the kernel code requires taking the cpu hotplug lock.  Other
      23             :  * parts of the kernel allocate memory with that lock held, which means that
      24             :  * XFS callers cannot hold any locks that might be used by memory reclaim or
      25             :  * writeback when calling the static_branch_{inc,dec} functions.
      26             :  */
      27             : static DEFINE_STATIC_KEY_FALSE(xfs_defer_drain_waiter_gate);
      28             : 
      29             : void
      30       69513 : xfs_defer_drain_wait_disable(void)
      31             : {
      32       69513 :         static_branch_dec(&xfs_defer_drain_waiter_gate);
      33       69513 : }
      34             : 
      35             : void
      36       69513 : xfs_defer_drain_wait_enable(void)
      37             : {
      38       69513 :         static_branch_inc(&xfs_defer_drain_waiter_gate);
      39       69513 : }
      40             : 
      41             : void
      42      135479 : xfs_defer_drain_init(
      43             :         struct xfs_defer_drain  *dr)
      44             : {
      45      135479 :         atomic_set(&dr->dr_count, 0);
      46      135479 :         init_waitqueue_head(&dr->dr_waiters);
      47      135479 : }
      48             : 
      49             : void
      50      135513 : xfs_defer_drain_free(struct xfs_defer_drain     *dr)
      51             : {
      52      135513 :         ASSERT(atomic_read(&dr->dr_count) == 0);
      53      135513 : }
      54             : 
      55             : /* Increase the pending intent count. */
      56   652617495 : static inline void xfs_defer_drain_grab(struct xfs_defer_drain *dr)
      57             : {
      58   652617495 :         atomic_inc(&dr->dr_count);
      59   652618666 : }
      60             : 
      61             : static inline bool has_waiters(struct wait_queue_head *wq_head)
      62             : {
      63             :         /*
      64             :          * This memory barrier is paired with the one in set_current_state on
      65             :          * the waiting side.
      66             :          */
      67     1301718 :         smp_mb__after_atomic();
      68     1301719 :         return waitqueue_active(wq_head);
      69             : }
      70             : 
      71             : /* Decrease the pending intent count, and wake any waiters, if appropriate. */
      72   652570284 : static inline void xfs_defer_drain_rele(struct xfs_defer_drain *dr)
      73             : {
      74  1305149417 :         if (atomic_dec_and_test(&dr->dr_count) &&
      75   258472897 :             static_branch_unlikely(&xfs_defer_drain_waiter_gate) &&
      76             :             has_waiters(&dr->dr_waiters))
      77       37736 :                 wake_up(&dr->dr_waiters);
      78   652579216 : }
      79             : 
      80             : /* Are there intents pending? */
      81             : static inline bool xfs_defer_drain_busy(struct xfs_defer_drain *dr)
      82             : {
      83     3354114 :         return atomic_read(&dr->dr_count) > 0;
      84             : }
      85             : 
      86             : /*
      87             :  * Wait for the pending intent count for a drain to hit zero.
      88             :  *
      89             :  * Callers must not hold any locks that would prevent intents from being
      90             :  * finished.
      91             :  */
      92       35310 : static inline int xfs_defer_drain_wait(struct xfs_defer_drain *dr)
      93             : {
      94       73029 :         return wait_event_killable(dr->dr_waiters, !xfs_defer_drain_busy(dr));
      95             : }
      96             : 
      97             : /*
      98             :  * Get a passive reference to an AG and declare an intent to update its
      99             :  * metadata.
     100             :  */
     101             : struct xfs_perag *
     102   510454865 : xfs_perag_intent_get(
     103             :         struct xfs_mount        *mp,
     104             :         xfs_agnumber_t          agno)
     105             : {
     106   510454865 :         struct xfs_perag        *pag;
     107             : 
     108   510454865 :         pag = xfs_perag_get(mp, agno);
     109   510473350 :         if (!pag)
     110             :                 return NULL;
     111             : 
     112   510473350 :         xfs_perag_intent_hold(pag);
     113   510473350 :         return pag;
     114             : }
     115             : 
     116             : /*
     117             :  * Release our intent to update this AG's metadata, and then release our
     118             :  * passive ref to the AG.
     119             :  */
     120             : void
     121   510418034 : xfs_perag_intent_put(
     122             :         struct xfs_perag        *pag)
     123             : {
     124   510418034 :         xfs_perag_intent_rele(pag);
     125   510422005 :         xfs_perag_put(pag);
     126   510424957 : }
     127             : 
     128             : /*
     129             :  * Declare an intent to update AG metadata.  Other threads that need exclusive
     130             :  * access can decide to back off if they see declared intentions.
     131             :  */
     132             : void
     133   510697608 : xfs_perag_intent_hold(
     134             :         struct xfs_perag        *pag)
     135             : {
     136   510697608 :         trace_xfs_perag_intent_hold(pag, __return_address);
     137   510704683 :         xfs_defer_drain_grab(&pag->pag_intents_drain);
     138   510702606 : }
     139             : 
     140             : /* Release our intent to update this AG's metadata. */
     141             : void
     142   510655087 : xfs_perag_intent_rele(
     143             :         struct xfs_perag        *pag)
     144             : {
     145   510655087 :         trace_xfs_perag_intent_rele(pag, __return_address);
     146   510659650 :         xfs_defer_drain_rele(&pag->pag_intents_drain);
     147   510658351 : }
     148             : 
     149             : /*
     150             :  * Wait for the intent update count for this AG to hit zero.
     151             :  * Callers must not hold any AG header buffers.
     152             :  */
     153             : int
     154       14598 : xfs_perag_intent_drain(
     155             :         struct xfs_perag        *pag)
     156             : {
     157       14598 :         trace_xfs_perag_wait_intents(pag, __return_address);
     158       14598 :         return xfs_defer_drain_wait(&pag->pag_intents_drain);
     159             : }
     160             : 
     161             : /* Has anyone declared an intent to update this AG? */
     162             : bool
     163     2935104 : xfs_perag_intent_busy(
     164             :         struct xfs_perag        *pag)
     165             : {
     166     2935104 :         return xfs_defer_drain_busy(&pag->pag_intents_drain);
     167             : }
     168             : 
     169             : #ifdef CONFIG_XFS_RT
     170             : 
     171             : /*
     172             :  * Get a passive reference to an rtgroup and declare an intent to update its
     173             :  * metadata.
     174             :  */
     175             : struct xfs_rtgroup *
     176   141913868 : xfs_rtgroup_intent_get(
     177             :         struct xfs_mount        *mp,
     178             :         xfs_rgnumber_t          rgno)
     179             : {
     180   141913868 :         struct xfs_rtgroup      *rtg;
     181             : 
     182   141913868 :         rtg = xfs_rtgroup_get(mp, rgno);
     183   141912806 :         if (!rtg)
     184             :                 return NULL;
     185             : 
     186   141912806 :         xfs_rtgroup_intent_hold(rtg);
     187   141912806 :         return rtg;
     188             : }
     189             : 
     190             : /*
     191             :  * Release our intent to update this rtgroup's metadata, and then release our
     192             :  * passive ref to the rtgroup.
     193             :  */
     194             : void
     195   141916661 : xfs_rtgroup_intent_put(
     196             :         struct xfs_rtgroup      *rtg)
     197             : {
     198   141916661 :         xfs_rtgroup_intent_rele(rtg);
     199   141918481 :         xfs_rtgroup_put(rtg);
     200   141918655 : }
     201             : /*
     202             :  * Declare an intent to update rtgroup metadata.  Other threads that need
     203             :  * exclusive access can decide to back off if they see declared intentions.
     204             :  */
     205             : void
     206   141913187 : xfs_rtgroup_intent_hold(
     207             :         struct xfs_rtgroup      *rtg)
     208             : {
     209   141913187 :         trace_xfs_rtgroup_intent_hold(rtg, __return_address);
     210   141913886 :         xfs_defer_drain_grab(&rtg->rtg_intents_drain);
     211   141913935 : }
     212             : 
     213             : /* Release our intent to update this rtgroup's metadata. */
     214             : void
     215   141916555 : xfs_rtgroup_intent_rele(
     216             :         struct xfs_rtgroup      *rtg)
     217             : {
     218   141916555 :         trace_xfs_rtgroup_intent_rele(rtg, __return_address);
     219   141916889 :         xfs_defer_drain_rele(&rtg->rtg_intents_drain);
     220   141918688 : }
     221             : 
     222             : /*
     223             :  * Wait for the intent update count for this rtgroup to hit zero.
     224             :  * Callers must not hold any rt metadata inode locks.
     225             :  */
     226             : int
     227       20712 : xfs_rtgroup_intent_drain(
     228             :         struct xfs_rtgroup      *rtg)
     229             : {
     230       20712 :         trace_xfs_rtgroup_wait_intents(rtg, __return_address);
     231       20712 :         return xfs_defer_drain_wait(&rtg->rtg_intents_drain);
     232             : }
     233             : 
     234             : /* Has anyone declared an intent to update this rtgroup? */
     235             : bool
     236      312310 : xfs_rtgroup_intent_busy(
     237             :         struct xfs_rtgroup      *rtg)
     238             : {
     239      312310 :         return xfs_defer_drain_busy(&rtg->rtg_intents_drain);
     240             : }
     241             : #endif /* CONFIG_XFS_RT */