Line data    Source code

       1             : // SPDX-License-Identifier: GPL-2.0-or-later
       2             : /*
       3             :  * Copyright (C) 2022-2023 Oracle.  All Rights Reserved.
       4             :  * Author: Darrick J. Wong <djwong@kernel.org>
       5             :  */
       6             : #include "xfs.h"
       7             : #include "xfs_fs.h"
       8             : #include "xfs_shared.h"
       9             : #include "xfs_format.h"
      10             : #include "xfs_trans_resv.h"
      11             : #include "xfs_mount.h"
      12             : #include "xfs_ag.h"
      13             : #include "xfs_trace.h"
      14             : #include "xfs_rtgroup.h"
      15             : 
      16             : /*
      17             :  * Use a static key here to reduce the overhead of xfs_defer_drain_rele.  If the
      18             :  * compiler supports jump labels, the static branch will be replaced by a nop
      19             :  * sled when there are no xfs_defer_drain_wait callers.  Online fsck is currently
      20             :  * the only caller, so this is a reasonable tradeoff.
      21             :  *
      22             :  * Note: Patching the kernel code requires taking the cpu hotplug lock.  Other
      23             :  * parts of the kernel allocate memory with that lock held, which means that
      24             :  * XFS callers cannot hold any locks that might be used by memory reclaim or
      25             :  * writeback when calling the static_branch_{inc,dec} functions.
      26             :  */
      27             : static DEFINE_STATIC_KEY_FALSE(xfs_defer_drain_waiter_gate);
      28             : 
      29             : void
      30      173576 : xfs_defer_drain_wait_disable(void)
      31             : {
      32      173576 :         static_branch_dec(&xfs_defer_drain_waiter_gate);
      33      173576 : }
      34             : 
      35             : void
      36      173576 : xfs_defer_drain_wait_enable(void)
      37             : {
      38      173576 :         static_branch_inc(&xfs_defer_drain_waiter_gate);
      39      173576 : }
      40             : 
      41             : void
      42      629355 : xfs_defer_drain_init(
      43             :         struct xfs_defer_drain  *dr)
      44             : {
      45      629355 :         atomic_set(&dr->dr_count, 0);
      46      629355 :         init_waitqueue_head(&dr->dr_waiters);
      47      629355 : }
      48             : 
      49             : void
      50      629419 : xfs_defer_drain_free(struct xfs_defer_drain     *dr)
      51             : {
      52      629419 :         ASSERT(atomic_read(&dr->dr_count) == 0);
      53      629419 : }
      54             : 
      55             : /* Increase the pending intent count. */
      56             : static inline void xfs_defer_drain_grab(struct xfs_defer_drain *dr)
      57             : {
      58  1500056946 :         atomic_inc(&dr->dr_count);
      59             : }
      60             : 
      61             : static inline bool has_waiters(struct wait_queue_head *wq_head)
      62             : {
      63             :         /*
      64             :          * This memory barrier is paired with the one in set_current_state on
      65             :          * the waiting side.
      66             :          */
      67     2194459 :         smp_mb__after_atomic();
      68     2194459 :         return waitqueue_active(wq_head);
      69             : }
      70             : 
      71             : /* Decrease the pending intent count, and wake any waiters, if appropriate. */
      72  1500115445 : static inline void xfs_defer_drain_rele(struct xfs_defer_drain *dr)
      73             : {
      74  1500115445 :         if (atomic_dec_and_test(&dr->dr_count) &&
      75   611595444 :             static_branch_unlikely(&xfs_defer_drain_waiter_gate) &&
      76             :             has_waiters(&dr->dr_waiters))
      77       98183 :                 wake_up(&dr->dr_waiters);
      78  1500194574 : }
      79             : 
      80             : /* Are there intents pending? */
      81             : static inline bool xfs_defer_drain_busy(struct xfs_defer_drain *dr)
      82             : {
      83     7120199 :         return atomic_read(&dr->dr_count) > 0;
      84             : }
      85             : 
      86             : /*
      87             :  * Wait for the pending intent count for a drain to hit zero.
      88             :  *
      89             :  * Callers must not hold any locks that would prevent intents from being
      90             :  * finished.
      91             :  */
      92       85941 : static inline int xfs_defer_drain_wait(struct xfs_defer_drain *dr)
      93             : {
      94      184096 :         return wait_event_killable(dr->dr_waiters, !xfs_defer_drain_busy(dr));
      95             : }
      96             : 
      97             : /*
      98             :  * Get a passive reference to an AG and declare an intent to update its
      99             :  * metadata.
     100             :  */
     101             : struct xfs_perag *
     102   930843865 : xfs_perag_intent_get(
     103             :         struct xfs_mount        *mp,
     104             :         xfs_agnumber_t          agno)
     105             : {
     106   930843865 :         struct xfs_perag        *pag;
     107             : 
     108   930843865 :         pag = xfs_perag_get(mp, agno);
     109   931096063 :         if (!pag)
     110             :                 return NULL;
     111             : 
     112   931096063 :         xfs_perag_intent_hold(pag);
     113   931096063 :         return pag;
     114             : }
     115             : 
     116             : /*
     117             :  * Release our intent to update this AG's metadata, and then release our
     118             :  * passive ref to the AG.
     119             :  */
     120             : void
     121   931160683 : xfs_perag_intent_put(
     122             :         struct xfs_perag        *pag)
     123             : {
     124   931160683 :         xfs_perag_intent_rele(pag);
     125   931161461 :         xfs_perag_put(pag);
     126   931181822 : }
     127             : 
     128             : /*
     129             :  * Declare an intent to update AG metadata.  Other threads that need exclusive
     130             :  * access can decide to back off if they see declared intentions.
     131             :  */
     132             : void
     133   931674632 : xfs_perag_intent_hold(
     134             :         struct xfs_perag        *pag)
     135             : {
     136   931674632 :         trace_xfs_perag_intent_hold(pag, __return_address);
     137   931629123 :         xfs_defer_drain_grab(&pag->pag_intents_drain);
     138   931718882 : }
     139             : 
     140             : /* Release our intent to update this AG's metadata. */
     141             : void
     142   931725933 : xfs_perag_intent_rele(
     143             :         struct xfs_perag        *pag)
     144             : {
     145   931725933 :         trace_xfs_perag_intent_rele(pag, __return_address);
     146   931696719 :         xfs_defer_drain_rele(&pag->pag_intents_drain);
     147   931752946 : }
     148             : 
     149             : /*
     150             :  * Wait for the intent update count for this AG to hit zero.
     151             :  * Callers must not hold any AG header buffers.
     152             :  */
     153             : int
     154       54855 : xfs_perag_intent_drain(
     155             :         struct xfs_perag        *pag)
     156             : {
     157       54855 :         trace_xfs_perag_wait_intents(pag, __return_address);
     158       54855 :         return xfs_defer_drain_wait(&pag->pag_intents_drain);
     159             : }
     160             : 
     161             : /* Has anyone declared an intent to update this AG? */
     162             : bool
     163     6266156 : xfs_perag_intent_busy(
     164             :         struct xfs_perag        *pag)
     165             : {
     166     6266156 :         return xfs_defer_drain_busy(&pag->pag_intents_drain);
     167             : }
     168             : 
     169             : #ifdef CONFIG_XFS_RT
     170             : 
     171             : /*
     172             :  * Get a passive reference to an rtgroup and declare an intent to update its
     173             :  * metadata.
     174             :  */
     175             : struct xfs_rtgroup *
     176   568428043 : xfs_rtgroup_intent_get(
     177             :         struct xfs_mount        *mp,
     178             :         xfs_rgnumber_t          rgno)
     179             : {
     180   568428043 :         struct xfs_rtgroup      *rtg;
     181             : 
     182   568428043 :         rtg = xfs_rtgroup_get(mp, rgno);
     183   568431790 :         if (!rtg)
     184             :                 return NULL;
     185             : 
     186   568431790 :         xfs_rtgroup_intent_hold(rtg);
     187   568431790 :         return rtg;
     188             : }
     189             : 
     190             : /*
     191             :  * Release our intent to update this rtgroup's metadata, and then release our
     192             :  * passive ref to the rtgroup.
     193             :  */
     194             : void
     195   568431507 : xfs_rtgroup_intent_put(
     196             :         struct xfs_rtgroup      *rtg)
     197             : {
     198   568431507 :         xfs_rtgroup_intent_rele(rtg);
     199   568430766 :         xfs_rtgroup_put(rtg);
     200   568429555 : }
     201             : /*
     202             :  * Declare an intent to update rtgroup metadata.  Other threads that need
     203             :  * exclusive access can decide to back off if they see declared intentions.
     204             :  */
     205             : void
     206   568430784 : xfs_rtgroup_intent_hold(
     207             :         struct xfs_rtgroup      *rtg)
     208             : {
     209   568430784 :         trace_xfs_rtgroup_intent_hold(rtg, __return_address);
     210   568427823 :         xfs_defer_drain_grab(&rtg->rtg_intents_drain);
     211   568433370 : }
     212             : 
     213             : /* Release our intent to update this rtgroup's metadata. */
     214             : void
     215   568429400 : xfs_rtgroup_intent_rele(
     216             :         struct xfs_rtgroup      *rtg)
     217             : {
     218   568429400 :         trace_xfs_rtgroup_intent_rele(rtg, __return_address);
     219   568425938 :         xfs_defer_drain_rele(&rtg->rtg_intents_drain);
     220   568431199 : }
     221             : 
     222             : /*
     223             :  * Wait for the intent update count for this rtgroup to hit zero.
     224             :  * Callers must not hold any rt metadata inode locks.
     225             :  */
     226             : int
     227       31086 : xfs_rtgroup_intent_drain(
     228             :         struct xfs_rtgroup      *rtg)
     229             : {
     230       31086 :         trace_xfs_rtgroup_wait_intents(rtg, __return_address);
     231       31086 :         return xfs_defer_drain_wait(&rtg->rtg_intents_drain);
     232             : }
     233             : 
     234             : /* Has anyone declared an intent to update this rtgroup? */
     235             : bool
     236      586001 : xfs_rtgroup_intent_busy(
     237             :         struct xfs_rtgroup      *rtg)
     238             : {
     239      586001 :         return xfs_defer_drain_busy(&rtg->rtg_intents_drain);
     240             : }
     241             : #endif /* CONFIG_XFS_RT */