diff options
| -rw-r--r-- | arch/s390/include/asm/spinlock.h | 50 | ||||
| -rw-r--r-- | arch/s390/lib/spinlock.c | 29 |
2 files changed, 46 insertions, 33 deletions
diff --git a/arch/s390/include/asm/spinlock.h b/arch/s390/include/asm/spinlock.h index 5a0b2882ad48..96879f7ad6da 100644 --- a/arch/s390/include/asm/spinlock.h +++ b/arch/s390/include/asm/spinlock.h @@ -59,7 +59,9 @@ static inline int arch_spin_is_locked(arch_spinlock_t *lp) static inline int arch_spin_trylock_once(arch_spinlock_t *lp) { - return _raw_compare_and_swap(&lp->lock, 0, SPINLOCK_LOCKVAL); + barrier(); + return likely(arch_spin_value_unlocked(*lp) && + _raw_compare_and_swap(&lp->lock, 0, SPINLOCK_LOCKVAL)); } static inline int arch_spin_tryrelease_once(arch_spinlock_t *lp) @@ -69,20 +71,20 @@ static inline int arch_spin_tryrelease_once(arch_spinlock_t *lp) static inline void arch_spin_lock(arch_spinlock_t *lp) { - if (unlikely(!arch_spin_trylock_once(lp))) + if (!arch_spin_trylock_once(lp)) arch_spin_lock_wait(lp); } static inline void arch_spin_lock_flags(arch_spinlock_t *lp, unsigned long flags) { - if (unlikely(!arch_spin_trylock_once(lp))) + if (!arch_spin_trylock_once(lp)) arch_spin_lock_wait_flags(lp, flags); } static inline int arch_spin_trylock(arch_spinlock_t *lp) { - if (unlikely(!arch_spin_trylock_once(lp))) + if (!arch_spin_trylock_once(lp)) return arch_spin_trylock_retry(lp); return 1; } @@ -128,19 +130,29 @@ extern void _raw_write_lock_wait(arch_rwlock_t *lp); extern void _raw_write_lock_wait_flags(arch_rwlock_t *lp, unsigned long flags); extern int _raw_write_trylock_retry(arch_rwlock_t *lp); +static inline int arch_read_trylock_once(arch_rwlock_t *rw) +{ + unsigned int old = ACCESS_ONCE(rw->lock); + return likely((int) old >= 0 && + _raw_compare_and_swap(&rw->lock, old, old + 1)); +} + +static inline int arch_write_trylock_once(arch_rwlock_t *rw) +{ + unsigned int old = ACCESS_ONCE(rw->lock); + return likely(old == 0 && + _raw_compare_and_swap(&rw->lock, 0, 0x80000000)); +} + static inline void arch_read_lock(arch_rwlock_t *rw) { - unsigned int old; - old = rw->lock & 0x7fffffffU; - if (!_raw_compare_and_swap(&rw->lock, old, old + 1)) + if (!arch_read_trylock_once(rw)) _raw_read_lock_wait(rw); } static inline void arch_read_lock_flags(arch_rwlock_t *rw, unsigned long flags) { - unsigned int old; - old = rw->lock & 0x7fffffffU; - if (!_raw_compare_and_swap(&rw->lock, old, old + 1)) + if (!arch_read_trylock_once(rw)) _raw_read_lock_wait_flags(rw, flags); } @@ -155,13 +167,13 @@ static inline void arch_read_unlock(arch_rwlock_t *rw) static inline void arch_write_lock(arch_rwlock_t *rw) { - if (unlikely(!_raw_compare_and_swap(&rw->lock, 0, 0x80000000))) + if (!arch_write_trylock_once(rw)) _raw_write_lock_wait(rw); } static inline void arch_write_lock_flags(arch_rwlock_t *rw, unsigned long flags) { - if (unlikely(!_raw_compare_and_swap(&rw->lock, 0, 0x80000000))) + if (!arch_write_trylock_once(rw)) _raw_write_lock_wait_flags(rw, flags); } @@ -172,18 +184,16 @@ static inline void arch_write_unlock(arch_rwlock_t *rw) static inline int arch_read_trylock(arch_rwlock_t *rw) { - unsigned int old; - old = rw->lock & 0x7fffffffU; - if (likely(_raw_compare_and_swap(&rw->lock, old, old + 1))) - return 1; - return _raw_read_trylock_retry(rw); + if (!arch_read_trylock_once(rw)) + return _raw_read_trylock_retry(rw); + return 1; } static inline int arch_write_trylock(arch_rwlock_t *rw) { - if (likely(_raw_compare_and_swap(&rw->lock, 0, 0x80000000))) - return 1; - return _raw_write_trylock_retry(rw); + if (!arch_write_trylock_once(rw)) + return _raw_write_trylock_retry(rw); + return 1; } #define arch_read_relax(lock) cpu_relax() diff --git a/arch/s390/lib/spinlock.c b/arch/s390/lib/spinlock.c index 3f0e682b7e62..284d879a9b8c 100644 --- a/arch/s390/lib/spinlock.c +++ b/arch/s390/lib/spinlock.c @@ -100,12 +100,9 @@ int arch_spin_trylock_retry(arch_spinlock_t *lp) { int count; - for (count = spin_retry; count > 0; count--) { - if (arch_spin_is_locked(lp)) - continue; + for (count = spin_retry; count > 0; count--) if (arch_spin_trylock_once(lp)) return 1; - } return 0; } EXPORT_SYMBOL(arch_spin_trylock_retry); @@ -120,9 +117,9 @@ void _raw_read_lock_wait(arch_rwlock_t *rw) smp_yield(); count = spin_retry; } - if (!arch_read_can_lock(rw)) + old = ACCESS_ONCE(rw->lock); + if ((int) old < 0) continue; - old = rw->lock & 0x7fffffffU; if (_raw_compare_and_swap(&rw->lock, old, old + 1)) return; } @@ -140,9 +137,9 @@ void _raw_read_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags) smp_yield(); count = spin_retry; } - if (!arch_read_can_lock(rw)) + old = ACCESS_ONCE(rw->lock); + if ((int) old < 0) continue; - old = rw->lock & 0x7fffffffU; local_irq_disable(); if (_raw_compare_and_swap(&rw->lock, old, old + 1)) return; @@ -156,9 +153,9 @@ int _raw_read_trylock_retry(arch_rwlock_t *rw) int count = spin_retry; while (count-- > 0) { - if (!arch_read_can_lock(rw)) + old = ACCESS_ONCE(rw->lock); + if ((int) old < 0) continue; - old = rw->lock & 0x7fffffffU; if (_raw_compare_and_swap(&rw->lock, old, old + 1)) return 1; } @@ -168,6 +165,7 @@ EXPORT_SYMBOL(_raw_read_trylock_retry); void _raw_write_lock_wait(arch_rwlock_t *rw) { + unsigned int old; int count = spin_retry; while (1) { @@ -175,7 +173,8 @@ void _raw_write_lock_wait(arch_rwlock_t *rw) smp_yield(); count = spin_retry; } - if (!arch_write_can_lock(rw)) + old = ACCESS_ONCE(rw->lock); + if (old) continue; if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000)) return; @@ -185,6 +184,7 @@ EXPORT_SYMBOL(_raw_write_lock_wait); void _raw_write_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags) { + unsigned int old; int count = spin_retry; local_irq_restore(flags); @@ -193,7 +193,8 @@ void _raw_write_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags) smp_yield(); count = spin_retry; } - if (!arch_write_can_lock(rw)) + old = ACCESS_ONCE(rw->lock); + if (old) continue; local_irq_disable(); if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000)) @@ -204,10 +205,12 @@ EXPORT_SYMBOL(_raw_write_lock_wait_flags); int _raw_write_trylock_retry(arch_rwlock_t *rw) { + unsigned int old; int count = spin_retry; while (count-- > 0) { - if (!arch_write_can_lock(rw)) + old = ACCESS_ONCE(rw->lock); + if (old) continue; if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000)) return 1; |