Line data    Source code

       1             : /* SPDX-License-Identifier: GPL-2.0-or-later */
       2             : /* delayacct.h - per-task delay accounting
       3             :  *
       4             :  * Copyright (C) Shailabh Nagar, IBM Corp. 2006
       5             :  */
       6             : 
       7             : #ifndef _LINUX_DELAYACCT_H
       8             : #define _LINUX_DELAYACCT_H
       9             : 
      10             : #include <uapi/linux/taskstats.h>
      11             : 
      12             : #ifdef CONFIG_TASK_DELAY_ACCT
      13             : struct task_delay_info {
      14             :         raw_spinlock_t  lock;
      15             : 
      16             :         /* For each stat XXX, add following, aligned appropriately
      17             :          *
      18             :          * struct timespec XXX_start, XXX_end;
      19             :          * u64 XXX_delay;
      20             :          * u32 XXX_count;
      21             :          *
      22             :          * Atomicity of updates to XXX_delay, XXX_count protected by
      23             :          * single lock above (split into XXX_lock if contention is an issue).
      24             :          */
      25             : 
      26             :         /*
      27             :          * XXX_count is incremented on every XXX operation, the delay
      28             :          * associated with the operation is added to XXX_delay.
      29             :          * XXX_delay contains the accumulated delay time in nanoseconds.
      30             :          */
      31             :         u64 blkio_start;
      32             :         u64 blkio_delay;        /* wait for sync block io completion */
      33             :         u64 swapin_start;
      34             :         u64 swapin_delay;       /* wait for swapin */
      35             :         u32 blkio_count;        /* total count of the number of sync block */
      36             :                                 /* io operations performed */
      37             :         u32 swapin_count;       /* total count of swapin */
      38             : 
      39             :         u64 freepages_start;
      40             :         u64 freepages_delay;    /* wait for memory reclaim */
      41             : 
      42             :         u64 thrashing_start;
      43             :         u64 thrashing_delay;    /* wait for thrashing page */
      44             : 
      45             :         u64 compact_start;
      46             :         u64 compact_delay;      /* wait for memory compact */
      47             : 
      48             :         u64 wpcopy_start;
      49             :         u64 wpcopy_delay;       /* wait for write-protect copy */
      50             : 
      51             :         u64 irq_delay;  /* wait for IRQ/SOFTIRQ */
      52             : 
      53             :         u32 freepages_count;    /* total count of memory reclaim */
      54             :         u32 thrashing_count;    /* total count of thrash waits */
      55             :         u32 compact_count;      /* total count of memory compact */
      56             :         u32 wpcopy_count;       /* total count of write-protect copy */
      57             :         u32 irq_count;  /* total count of IRQ/SOFTIRQ */
      58             : };
      59             : #endif
      60             : 
      61             : #include <linux/sched.h>
      62             : #include <linux/slab.h>
      63             : #include <linux/jump_label.h>
      64             : 
      65             : #ifdef CONFIG_TASK_DELAY_ACCT
      66             : DECLARE_STATIC_KEY_FALSE(delayacct_key);
      67             : extern int delayacct_on;        /* Delay accounting turned on/off */
      68             : extern struct kmem_cache *delayacct_cache;
      69             : extern void delayacct_init(void);
      70             : 
      71             : extern void __delayacct_tsk_init(struct task_struct *);
      72             : extern void __delayacct_tsk_exit(struct task_struct *);
      73             : extern void __delayacct_blkio_start(void);
      74             : extern void __delayacct_blkio_end(struct task_struct *);
      75             : extern int delayacct_add_tsk(struct taskstats *, struct task_struct *);
      76             : extern __u64 __delayacct_blkio_ticks(struct task_struct *);
      77             : extern void __delayacct_freepages_start(void);
      78             : extern void __delayacct_freepages_end(void);
      79             : extern void __delayacct_thrashing_start(bool *in_thrashing);
      80             : extern void __delayacct_thrashing_end(bool *in_thrashing);
      81             : extern void __delayacct_swapin_start(void);
      82             : extern void __delayacct_swapin_end(void);
      83             : extern void __delayacct_compact_start(void);
      84             : extern void __delayacct_compact_end(void);
      85             : extern void __delayacct_wpcopy_start(void);
      86             : extern void __delayacct_wpcopy_end(void);
      87             : extern void __delayacct_irq(struct task_struct *task, u32 delta);
      88             : 
      89             : static inline void delayacct_tsk_init(struct task_struct *tsk)
      90             : {
      91             :         /* reinitialize in case parent's non-null pointer was dup'ed*/
      92             :         tsk->delays = NULL;
      93             :         if (delayacct_on)
      94             :                 __delayacct_tsk_init(tsk);
      95             : }
      96             : 
      97             : /* Free tsk->delays. Called from bad fork and __put_task_struct
      98             :  * where there's no risk of tsk->delays being accessed elsewhere
      99             :  */
     100             : static inline void delayacct_tsk_free(struct task_struct *tsk)
     101             : {
     102             :         if (tsk->delays)
     103             :                 kmem_cache_free(delayacct_cache, tsk->delays);
     104             :         tsk->delays = NULL;
     105             : }
     106             : 
     107             : static inline void delayacct_blkio_start(void)
     108             : {
     109             :         if (!static_branch_unlikely(&delayacct_key))
     110             :                 return;
     111             : 
     112             :         if (current->delays)
     113             :                 __delayacct_blkio_start();
     114             : }
     115             : 
     116             : static inline void delayacct_blkio_end(struct task_struct *p)
     117             : {
     118             :         if (!static_branch_unlikely(&delayacct_key))
     119             :                 return;
     120             : 
     121             :         if (p->delays)
     122             :                 __delayacct_blkio_end(p);
     123             : }
     124             : 
     125             : static inline __u64 delayacct_blkio_ticks(struct task_struct *tsk)
     126             : {
     127             :         if (tsk->delays)
     128             :                 return __delayacct_blkio_ticks(tsk);
     129             :         return 0;
     130             : }
     131             : 
     132             : static inline void delayacct_freepages_start(void)
     133             : {
     134             :         if (!static_branch_unlikely(&delayacct_key))
     135             :                 return;
     136             : 
     137             :         if (current->delays)
     138             :                 __delayacct_freepages_start();
     139             : }
     140             : 
     141             : static inline void delayacct_freepages_end(void)
     142             : {
     143             :         if (!static_branch_unlikely(&delayacct_key))
     144             :                 return;
     145             : 
     146             :         if (current->delays)
     147             :                 __delayacct_freepages_end();
     148             : }
     149             : 
     150      406012 : static inline void delayacct_thrashing_start(bool *in_thrashing)
     151             : {
     152      406012 :         if (!static_branch_unlikely(&delayacct_key))
     153             :                 return;
     154             : 
     155           0 :         if (current->delays)
     156           0 :                 __delayacct_thrashing_start(in_thrashing);
     157             : }
     158             : 
     159      406026 : static inline void delayacct_thrashing_end(bool *in_thrashing)
     160             : {
     161      406026 :         if (!static_branch_unlikely(&delayacct_key))
     162             :                 return;
     163             : 
     164           0 :         if (current->delays)
     165           0 :                 __delayacct_thrashing_end(in_thrashing);
     166             : }
     167             : 
     168             : static inline void delayacct_swapin_start(void)
     169             : {
     170             :         if (!static_branch_unlikely(&delayacct_key))
     171             :                 return;
     172             : 
     173             :         if (current->delays)
     174             :                 __delayacct_swapin_start();
     175             : }
     176             : 
     177             : static inline void delayacct_swapin_end(void)
     178             : {
     179             :         if (!static_branch_unlikely(&delayacct_key))
     180             :                 return;
     181             : 
     182             :         if (current->delays)
     183             :                 __delayacct_swapin_end();
     184             : }
     185             : 
     186             : static inline void delayacct_compact_start(void)
     187             : {
     188             :         if (!static_branch_unlikely(&delayacct_key))
     189             :                 return;
     190             : 
     191             :         if (current->delays)
     192             :                 __delayacct_compact_start();
     193             : }
     194             : 
     195             : static inline void delayacct_compact_end(void)
     196             : {
     197             :         if (!static_branch_unlikely(&delayacct_key))
     198             :                 return;
     199             : 
     200             :         if (current->delays)
     201             :                 __delayacct_compact_end();
     202             : }
     203             : 
     204             : static inline void delayacct_wpcopy_start(void)
     205             : {
     206             :         if (!static_branch_unlikely(&delayacct_key))
     207             :                 return;
     208             : 
     209             :         if (current->delays)
     210             :                 __delayacct_wpcopy_start();
     211             : }
     212             : 
     213             : static inline void delayacct_wpcopy_end(void)
     214             : {
     215             :         if (!static_branch_unlikely(&delayacct_key))
     216             :                 return;
     217             : 
     218             :         if (current->delays)
     219             :                 __delayacct_wpcopy_end();
     220             : }
     221             : 
     222             : static inline void delayacct_irq(struct task_struct *task, u32 delta)
     223             : {
     224             :         if (!static_branch_unlikely(&delayacct_key))
     225             :                 return;
     226             : 
     227             :         if (task->delays)
     228             :                 __delayacct_irq(task, delta);
     229             : }
     230             : 
     231             : #else
     232             : static inline void delayacct_init(void)
     233             : {}
     234             : static inline void delayacct_tsk_init(struct task_struct *tsk)
     235             : {}
     236             : static inline void delayacct_tsk_free(struct task_struct *tsk)
     237             : {}
     238             : static inline void delayacct_blkio_start(void)
     239             : {}
     240             : static inline void delayacct_blkio_end(struct task_struct *p)
     241             : {}
     242             : static inline int delayacct_add_tsk(struct taskstats *d,
     243             :                                         struct task_struct *tsk)
     244             : { return 0; }
     245             : static inline __u64 delayacct_blkio_ticks(struct task_struct *tsk)
     246             : { return 0; }
     247             : static inline int delayacct_is_task_waiting_on_io(struct task_struct *p)
     248             : { return 0; }
     249             : static inline void delayacct_freepages_start(void)
     250             : {}
     251             : static inline void delayacct_freepages_end(void)
     252             : {}
     253             : static inline void delayacct_thrashing_start(bool *in_thrashing)
     254             : {}
     255             : static inline void delayacct_thrashing_end(bool *in_thrashing)
     256             : {}
     257             : static inline void delayacct_swapin_start(void)
     258             : {}
     259             : static inline void delayacct_swapin_end(void)
     260             : {}
     261             : static inline void delayacct_compact_start(void)
     262             : {}
     263             : static inline void delayacct_compact_end(void)
     264             : {}
     265             : static inline void delayacct_wpcopy_start(void)
     266             : {}
     267             : static inline void delayacct_wpcopy_end(void)
     268             : {}
     269             : static inline void delayacct_irq(struct task_struct *task, u32 delta)
     270             : {}
     271             : 
     272             : #endif /* CONFIG_TASK_DELAY_ACCT */
     273             : 
     274             : #endif