Line data    Source code

       1             : /* SPDX-License-Identifier: GPL-2.0 */
       2             : #ifndef __LINUX_BIT_SPINLOCK_H
       3             : #define __LINUX_BIT_SPINLOCK_H
       4             : 
       5             : #include <linux/kernel.h>
       6             : #include <linux/preempt.h>
       7             : #include <linux/atomic.h>
       8             : #include <linux/bug.h>
       9             : 
      10             : /*
      11             :  *  bit-based spin_lock()
      12             :  *
      13             :  * Don't use this unless you really need to: spin_lock() and spin_unlock()
      14             :  * are significantly faster.
      15             :  */
      16  3830228913 : static inline void bit_spin_lock(int bitnum, unsigned long *addr)
      17             : {
      18             :         /*
      19             :          * Assuming the lock is uncontended, this never enters
      20             :          * the body of the outer loop. If it is contended, then
      21             :          * within the inner loop a non-atomic test is used to
      22             :          * busywait with less bus contention for a good time to
      23             :          * attempt to acquire the lock bit.
      24             :          */
      25  3830228913 :         preempt_disable();
      26             : #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
      27  7665575397 :         while (unlikely(test_and_set_bit_lock(bitnum, addr))) {
      28     3445771 :                 preempt_enable();
      29     6569359 :                 do {
      30     6569359 :                         cpu_relax();
      31    13135304 :                 } while (test_bit(bitnum, addr));
      32  3833764664 :                 preempt_disable();
      33             :         }
      34             : #endif
      35  3831769102 :         __acquire(bitlock);
      36  3831769102 : }
      37             : 
      38             : /*
      39             :  * Return true if it was acquired
      40             :  */
      41             : static inline int bit_spin_trylock(int bitnum, unsigned long *addr)
      42             : {
      43             :         preempt_disable();
      44             : #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
      45             :         if (unlikely(test_and_set_bit_lock(bitnum, addr))) {
      46             :                 preempt_enable();
      47             :                 return 0;
      48             :         }
      49             : #endif
      50             :         __acquire(bitlock);
      51             :         return 1;
      52             : }
      53             : 
      54             : /*
      55             :  *  bit-based spin_unlock()
      56             :  */
      57    60613931 : static inline void bit_spin_unlock(int bitnum, unsigned long *addr)
      58             : {
      59             : #ifdef CONFIG_DEBUG_SPINLOCK
      60   121227791 :         BUG_ON(!test_bit(bitnum, addr));
      61             : #endif
      62             : #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
      63    60613860 :         clear_bit_unlock(bitnum, addr);
      64             : #endif
      65    60616234 :         preempt_enable();
      66    60616195 :         __release(bitlock);
      67    60616195 : }
      68             : 
      69             : /*
      70             :  *  bit-based spin_unlock()
      71             :  *  non-atomic version, which can be used eg. if the bit lock itself is
      72             :  *  protecting the rest of the flags in the word.
      73             :  */
      74  3665020496 : static inline void __bit_spin_unlock(int bitnum, unsigned long *addr)
      75             : {
      76             : #ifdef CONFIG_DEBUG_SPINLOCK
      77  7330078733 :         BUG_ON(!test_bit(bitnum, addr));
      78             : #endif
      79             : #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
      80  3665058237 :         __clear_bit_unlock(bitnum, addr);
      81             : #endif
      82  3665069334 :         preempt_enable();
      83  3664982211 :         __release(bitlock);
      84  3664982211 : }
      85             : 
      86             : /*
      87             :  * Return true if the lock is held.
      88             :  */
      89             : static inline int bit_spin_is_locked(int bitnum, unsigned long *addr)
      90             : {
      91             : #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
      92             :         return test_bit(bitnum, addr);
      93             : #elif defined CONFIG_PREEMPT_COUNT
      94             :         return preempt_count();
      95             : #else
      96             :         return 1;
      97             : #endif
      98             : }
      99             : 
     100             : #endif /* __LINUX_BIT_SPINLOCK_H */
     101             :