Line data    Source code

       1             : /* SPDX-License-Identifier: GPL-2.0 */
       2             : #ifndef __LINUX_BIT_SPINLOCK_H
       3             : #define __LINUX_BIT_SPINLOCK_H
       4             : 
       5             : #include <linux/kernel.h>
       6             : #include <linux/preempt.h>
       7             : #include <linux/atomic.h>
       8             : #include <linux/bug.h>
       9             : 
      10             : /*
      11             :  *  bit-based spin_lock()
      12             :  *
      13             :  * Don't use this unless you really need to: spin_lock() and spin_unlock()
      14             :  * are significantly faster.
      15             :  */
      16  5106827921 : static inline void bit_spin_lock(int bitnum, unsigned long *addr)
      17             : {
      18             :         /*
      19             :          * Assuming the lock is uncontended, this never enters
      20             :          * the body of the outer loop. If it is contended, then
      21             :          * within the inner loop a non-atomic test is used to
      22             :          * busywait with less bus contention for a good time to
      23             :          * attempt to acquire the lock bit.
      24             :          */
      25  5106827921 :         preempt_disable();
      26             : #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
      27 10219094466 :         while (unlikely(test_and_set_bit_lock(bitnum, addr))) {
      28     3372677 :                 preempt_enable();
      29     6681093 :                 do {
      30     6681093 :                         cpu_relax();
      31    13360692 :                 } while (test_bit(bitnum, addr));
      32  5110208460 :                 preempt_disable();
      33             :         }
      34             : #endif
      35  5108833840 :         __acquire(bitlock);
      36  5108833840 : }
      37             : 
      38             : /*
      39             :  * Return true if it was acquired
      40             :  */
      41             : static inline int bit_spin_trylock(int bitnum, unsigned long *addr)
      42             : {
      43             :         preempt_disable();
      44             : #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
      45             :         if (unlikely(test_and_set_bit_lock(bitnum, addr))) {
      46             :                 preempt_enable();
      47             :                 return 0;
      48             :         }
      49             : #endif
      50             :         __acquire(bitlock);
      51             :         return 1;
      52             : }
      53             : 
      54             : /*
      55             :  *  bit-based spin_unlock()
      56             :  */
      57    75827504 : static inline void bit_spin_unlock(int bitnum, unsigned long *addr)
      58             : {
      59             : #ifdef CONFIG_DEBUG_SPINLOCK
      60   151654916 :         BUG_ON(!test_bit(bitnum, addr));
      61             : #endif
      62             : #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
      63    75827412 :         clear_bit_unlock(bitnum, addr);
      64             : #endif
      65    75833290 :         preempt_enable();
      66    75832633 :         __release(bitlock);
      67    75832633 : }
      68             : 
      69             : /*
      70             :  *  bit-based spin_unlock()
      71             :  *  non-atomic version, which can be used eg. if the bit lock itself is
      72             :  *  protecting the rest of the flags in the word.
      73             :  */
      74  4857253212 : static inline void __bit_spin_unlock(int bitnum, unsigned long *addr)
      75             : {
      76             : #ifdef CONFIG_DEBUG_SPINLOCK
      77  9714487620 :         BUG_ON(!test_bit(bitnum, addr));
      78             : #endif
      79             : #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
      80  4857234408 :         __clear_bit_unlock(bitnum, addr);
      81             : #endif
      82  4857292378 :         preempt_enable();
      83  4857263569 :         __release(bitlock);
      84  4857263569 : }
      85             : 
      86             : /*
      87             :  * Return true if the lock is held.
      88             :  */
      89             : static inline int bit_spin_is_locked(int bitnum, unsigned long *addr)
      90             : {
      91             : #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
      92             :         return test_bit(bitnum, addr);
      93             : #elif defined CONFIG_PREEMPT_COUNT
      94             :         return preempt_count();
      95             : #else
      96             :         return 1;
      97             : #endif
      98             : }
      99             : 
     100             : #endif /* __LINUX_BIT_SPINLOCK_H */
     101             :