Line data    Source code

       1             : /* SPDX-License-Identifier: GPL-2.0 */
       2             : #ifndef _LINUX_PERCPU_RWSEM_H
       3             : #define _LINUX_PERCPU_RWSEM_H
       4             : 
       5             : #include <linux/atomic.h>
       6             : #include <linux/percpu.h>
       7             : #include <linux/rcuwait.h>
       8             : #include <linux/wait.h>
       9             : #include <linux/rcu_sync.h>
      10             : #include <linux/lockdep.h>
      11             : 
      12             : struct percpu_rw_semaphore {
      13             :         struct rcu_sync         rss;
      14             :         unsigned int __percpu   *read_count;
      15             :         struct rcuwait          writer;
      16             :         wait_queue_head_t       waiters;
      17             :         atomic_t                block;
      18             : #ifdef CONFIG_DEBUG_LOCK_ALLOC
      19             :         struct lockdep_map      dep_map;
      20             : #endif
      21             : };
      22             : 
      23             : #ifdef CONFIG_DEBUG_LOCK_ALLOC
      24             : #define __PERCPU_RWSEM_DEP_MAP_INIT(lockname)   .dep_map = { .name = #lockname },
      25             : #else
      26             : #define __PERCPU_RWSEM_DEP_MAP_INIT(lockname)
      27             : #endif
      28             : 
      29             : #define __DEFINE_PERCPU_RWSEM(name, is_static)                          \
      30             : static DEFINE_PER_CPU(unsigned int, __percpu_rwsem_rc_##name);          \
      31             : is_static struct percpu_rw_semaphore name = {                           \
      32             :         .rss = __RCU_SYNC_INITIALIZER(name.rss),                        \
      33             :         .read_count = &__percpu_rwsem_rc_##name,                    \
      34             :         .writer = __RCUWAIT_INITIALIZER(name.writer),                   \
      35             :         .waiters = __WAIT_QUEUE_HEAD_INITIALIZER(name.waiters),         \
      36             :         .block = ATOMIC_INIT(0),                                        \
      37             :         __PERCPU_RWSEM_DEP_MAP_INIT(name)                               \
      38             : }
      39             : 
      40             : #define DEFINE_PERCPU_RWSEM(name)               \
      41             :         __DEFINE_PERCPU_RWSEM(name, /* not static */)
      42             : #define DEFINE_STATIC_PERCPU_RWSEM(name)        \
      43             :         __DEFINE_PERCPU_RWSEM(name, static)
      44             : 
      45             : extern bool __percpu_down_read(struct percpu_rw_semaphore *, bool);
      46             : 
      47  2217069581 : static inline void percpu_down_read(struct percpu_rw_semaphore *sem)
      48             : {
      49  2217069581 :         might_sleep();
      50             : 
      51  2217311839 :         rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);
      52             : 
      53  2217311839 :         preempt_disable();
      54             :         /*
      55             :          * We are in an RCU-sched read-side critical section, so the writer
      56             :          * cannot both change sem->state from readers_fast and start checking
      57             :          * counters while we are here. So if we see !sem->state, we know that
      58             :          * the writer won't be checking until we're past the preempt_enable()
      59             :          * and that once the synchronize_rcu() is done, the writer will see
      60             :          * anything we did within this RCU-sched read-size critical section.
      61             :          */
      62  2217398233 :         if (likely(rcu_sync_is_idle(&sem->rss)))
      63  2180099434 :                 this_cpu_inc(*sem->read_count);
      64             :         else
      65    37298799 :                 __percpu_down_read(sem, false); /* Unconditional memory barrier */
      66             :         /*
      67             :          * The preempt_enable() prevents the compiler from
      68             :          * bleeding the critical section out.
      69             :          */
      70  2217469685 :         preempt_enable();
      71  2217404725 : }
      72             : 
      73    47294443 : static inline bool percpu_down_read_trylock(struct percpu_rw_semaphore *sem)
      74             : {
      75    47294443 :         bool ret = true;
      76             : 
      77    47294443 :         preempt_disable();
      78             :         /*
      79             :          * Same as in percpu_down_read().
      80             :          */
      81    47294861 :         if (likely(rcu_sync_is_idle(&sem->rss)))
      82    45750518 :                 this_cpu_inc(*sem->read_count);
      83             :         else
      84     1544343 :                 ret = __percpu_down_read(sem, true); /* Unconditional memory barrier */
      85    47294606 :         preempt_enable();
      86             :         /*
      87             :          * The barrier() from preempt_enable() prevents the compiler from
      88             :          * bleeding the critical section out.
      89             :          */
      90             : 
      91    47295218 :         if (ret)
      92    47295218 :                 rwsem_acquire_read(&sem->dep_map, 0, 1, _RET_IP_);
      93             : 
      94    47295218 :         return ret;
      95             : }
      96             : 
      97  2264336278 : static inline void percpu_up_read(struct percpu_rw_semaphore *sem)
      98             : {
      99  2264336278 :         rwsem_release(&sem->dep_map, _RET_IP_);
     100             : 
     101  2264336278 :         preempt_disable();
     102             :         /*
     103             :          * Same as in percpu_down_read().
     104             :          */
     105  2264616508 :         if (likely(rcu_sync_is_idle(&sem->rss))) {
     106  2225933582 :                 this_cpu_dec(*sem->read_count);
     107             :         } else {
     108             :                 /*
     109             :                  * slowpath; reader will only ever wake a single blocked
     110             :                  * writer.
     111             :                  */
     112    38682926 :                 smp_mb(); /* B matches C */
     113             :                 /*
     114             :                  * In other words, if they see our decrement (presumably to
     115             :                  * aggregate zero, as that is the only time it matters) they
     116             :                  * will also see our critical section.
     117             :                  */
     118    38685798 :                 this_cpu_dec(*sem->read_count);
     119    38681567 :                 rcuwait_wake_up(&sem->writer);
     120             :         }
     121  2264646617 :         preempt_enable();
     122  2264622752 : }
     123             : 
     124             : extern bool percpu_is_read_locked(struct percpu_rw_semaphore *);
     125             : extern void percpu_down_write(struct percpu_rw_semaphore *);
     126             : extern void percpu_up_write(struct percpu_rw_semaphore *);
     127             : 
     128             : static inline bool percpu_is_write_locked(struct percpu_rw_semaphore *sem)
     129             : {
     130             :         return atomic_read(&sem->block);
     131             : }
     132             : 
     133             : extern int __percpu_init_rwsem(struct percpu_rw_semaphore *,
     134             :                                 const char *, struct lock_class_key *);
     135             : 
     136             : extern void percpu_free_rwsem(struct percpu_rw_semaphore *);
     137             : 
     138             : #define percpu_init_rwsem(sem)                                  \
     139             : ({                                                              \
     140             :         static struct lock_class_key rwsem_key;                 \
     141             :         __percpu_init_rwsem(sem, #sem, &rwsem_key);         \
     142             : })
     143             : 
     144             : #define percpu_rwsem_is_held(sem)       lockdep_is_held(sem)
     145             : #define percpu_rwsem_assert_held(sem)   lockdep_assert_held(sem)
     146             : 
     147             : static inline void percpu_rwsem_release(struct percpu_rw_semaphore *sem,
     148             :                                         bool read, unsigned long ip)
     149             : {
     150             :         lock_release(&sem->dep_map, ip);
     151             : }
     152             : 
     153             : static inline void percpu_rwsem_acquire(struct percpu_rw_semaphore *sem,
     154             :                                         bool read, unsigned long ip)
     155             : {
     156             :         lock_acquire(&sem->dep_map, 0, 1, read, 1, NULL, ip);
     157             : }
     158             : 
     159             : #endif