seqlock.h

Go to the documentation of this file.

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_SEQLOCK_H
#define __LINUX_SEQLOCK_H
 
/*
 * seqcount_t / seqlock_t - a reader-writer consistency mechanism with
 * lockless readers (read-only retry loops), and no writer starvation.
 *
 * See Documentation/locking/seqlock.rst
 *
 * Copyrights:
 * - Based on x86_64 vsyscall gettimeofday: Keith Owens, Andrea Arcangeli
 * - Sequence counters with associated locks, (C) 2020 Linutronix GmbH
 */
 
#include <linux/compiler.h>
#include <linux/cleanup.h>
#include <linux/kcsan-checks.h>
#include <linux/lockdep.h>
#include <linux/mutex.h>
#include <linux/preempt.h>
#include <linux/seqlock_types.h>
#include <linux/spinlock.h>
 
#include <asm/processor.h>
 
/*
 * The seqlock seqcount_t interface does not prescribe a precise sequence of
 * read begin/retry/end. For readers, typically there is a call to
 * read_seqcount_begin() and read_seqcount_retry(), however, there are more
 * esoteric cases which do not follow this pattern.
 *
 * As a consequence, we take the following best-effort approach for raw usage
 * via seqcount_t under KCSAN: upon beginning a seq-reader critical section,
 * pessimistically mark the next KCSAN_SEQLOCK_REGION_MAX memory accesses as
 * atomics; if there is a matching read_seqcount_retry() call, no following
 * memory operations are considered atomic. Usage of the seqlock_t interface
 * is not affected.
 */
#define KCSAN_SEQLOCK_REGION_MAX 1000
 
static inline void __seqcount_init(seqcount_t *s, const char *name,
                      struct lock_class_key *key)
{
    /*
     * Make sure we are not reinitializing a held lock:
     */
    lockdep_init_map(&s->dep_map, name, key, 0);
    s->sequence = 0;
}
 
#ifdef CONFIG_DEBUG_LOCK_ALLOC
 
# define SEQCOUNT_DEP_MAP_INIT(lockname)                \
        .dep_map = { .name = #lockname }
 
# define seqcount_init(s)                       \
    do {                                \
        static struct lock_class_key __key;         \
        __seqcount_init((s), #s, &__key);           \
    } while (0)
 
static inline void seqcount_lockdep_reader_access(const seqcount_t *s)
{
    seqcount_t *l = (seqcount_t *)s;
    unsigned long flags;
 
    local_irq_save(flags);
    seqcount_acquire_read(&l->dep_map, 0, 0, _RET_IP_);
    seqcount_release(&l->dep_map, _RET_IP_);
    local_irq_restore(flags);
}
 
#else
# define SEQCOUNT_DEP_MAP_INIT(lockname)
# define seqcount_init(s) __seqcount_init(s, NULL, NULL)
# define seqcount_lockdep_reader_access(x)
#endif
 
#define SEQCNT_ZERO(name) { .sequence = 0, SEQCOUNT_DEP_MAP_INIT(name) }
 
/*
 * Sequence counters with associated locks (seqcount_LOCKNAME_t)
 *
 * A sequence counter which associates the lock used for writer
 * serialization at initialization time. This enables lockdep to validate
 * that the write side critical section is properly serialized.
 *
 * For associated locks which do not implicitly disable preemption,
 * preemption protection is enforced in the write side function.
 *
 * Lockdep is never used in any for the raw write variants.
 *
 * See Documentation/locking/seqlock.rst
 */
 
/*
 * typedef seqcount_LOCKNAME_t - sequence counter with LOCKNAME associated
 * @seqcount:   The real sequence counter
 * @lock:   Pointer to the associated lock
 *
 * A plain sequence counter with external writer synchronization by
 * LOCKNAME @lock. The lock is associated to the sequence counter in the
 * static initializer or init function. This enables lockdep to validate
 * that the write side critical section is properly serialized.
 *
 * LOCKNAME:    raw_spinlock, spinlock, rwlock or mutex
 */
 
/*
 * seqcount_LOCKNAME_init() - runtime initializer for seqcount_LOCKNAME_t
 * @s:      Pointer to the seqcount_LOCKNAME_t instance
 * @lock:   Pointer to the associated lock
 */
 
#define seqcount_LOCKNAME_init(s, _lock, lockname)          \
    do {                                \
        seqcount_##lockname##_t *____s = (s);           \
        seqcount_init(&____s->seqcount);            \
        __SEQ_LOCK(____s->lock = (_lock));          \
    } while (0)
 
#define seqcount_raw_spinlock_init(s, lock) seqcount_LOCKNAME_init(s, lock, raw_spinlock)
#define seqcount_spinlock_init(s, lock)     seqcount_LOCKNAME_init(s, lock, spinlock)
#define seqcount_rwlock_init(s, lock)       seqcount_LOCKNAME_init(s, lock, rwlock)
#define seqcount_mutex_init(s, lock)        seqcount_LOCKNAME_init(s, lock, mutex)
 
/*
 * SEQCOUNT_LOCKNAME()  - Instantiate seqcount_LOCKNAME_t and helpers
 * seqprop_LOCKNAME_*() - Property accessors for seqcount_LOCKNAME_t
 *
 * @lockname:       "LOCKNAME" part of seqcount_LOCKNAME_t
 * @locktype:       LOCKNAME canonical C data type
 * @preemptible:    preemptibility of above locktype
 * @lockbase:       prefix for associated lock/unlock
 */
#define SEQCOUNT_LOCKNAME(lockname, locktype, preemptible, lockbase)    \
static __always_inline seqcount_t *                 \
__seqprop_##lockname##_ptr(seqcount_##lockname##_t *s)          \
{                                   \
    return &s->seqcount;                        \
}                                   \
                                    \
static __always_inline const seqcount_t *               \
__seqprop_##lockname##_const_ptr(const seqcount_##lockname##_t *s)  \
{                                   \
    return &s->seqcount;                        \
}                                   \
                                    \
static __always_inline unsigned                     \
__seqprop_##lockname##_sequence(const seqcount_##lockname##_t *s)   \
{                                   \
    unsigned seq = smp_load_acquire(&s->seqcount.sequence);     \
                                    \
    if (!IS_ENABLED(CONFIG_PREEMPT_RT))             \
        return seq;                     \
                                    \
    if (preemptible && unlikely(seq & 1)) {             \
        __SEQ_LOCK(lockbase##_lock(s->lock));           \
        __SEQ_LOCK(lockbase##_unlock(s->lock));         \
                                    \
        /*                          \
         * Re-read the sequence counter since the (possibly \
         * preempted) writer made progress.         \
         */                         \
        seq = smp_load_acquire(&s->seqcount.sequence);      \
    }                               \
                                    \
    return seq;                         \
}                                   \
                                    \
static __always_inline bool                     \
__seqprop_##lockname##_preemptible(const seqcount_##lockname##_t *s)    \
{                                   \
    if (!IS_ENABLED(CONFIG_PREEMPT_RT))             \
        return preemptible;                 \
                                    \
    /* PREEMPT_RT relies on the above LOCK+UNLOCK */        \
    return false;                           \
}                                   \
                                    \
static __always_inline void                     \
__seqprop_##lockname##_assert(const seqcount_##lockname##_t *s)     \
{                                   \
    __SEQ_LOCK(lockdep_assert_held(s->lock));           \
}
 
/*
 * __seqprop() for seqcount_t
 */
 
static inline seqcount_t *__seqprop_ptr(seqcount_t *s)
{
    return s;
}
 
static inline const seqcount_t *__seqprop_const_ptr(const seqcount_t *s)
{
    return s;
}
 
static inline unsigned __seqprop_sequence(const seqcount_t *s)
{
    return smp_load_acquire(&s->sequence);
}
 
static inline bool __seqprop_preemptible(const seqcount_t *s)
{
    return false;
}
 
static inline void __seqprop_assert(const seqcount_t *s)
{
    lockdep_assert_preemption_disabled();
}
 
#define __SEQ_RT    IS_ENABLED(CONFIG_PREEMPT_RT)
 
SEQCOUNT_LOCKNAME(raw_spinlock, raw_spinlock_t,  false,    raw_spin)
SEQCOUNT_LOCKNAME(spinlock,     spinlock_t,      __SEQ_RT, spin)
SEQCOUNT_LOCKNAME(rwlock,       rwlock_t,        __SEQ_RT, read)
SEQCOUNT_LOCKNAME(mutex,        struct mutex,    true,     mutex)
#undef SEQCOUNT_LOCKNAME
 
/*
 * SEQCNT_LOCKNAME_ZERO - static initializer for seqcount_LOCKNAME_t
 * @name:   Name of the seqcount_LOCKNAME_t instance
 * @lock:   Pointer to the associated LOCKNAME
 */
 
#define SEQCOUNT_LOCKNAME_ZERO(seq_name, assoc_lock) {          \
    .seqcount       = SEQCNT_ZERO(seq_name.seqcount),   \
    __SEQ_LOCK(.lock    = (assoc_lock))             \
}
 
#define SEQCNT_RAW_SPINLOCK_ZERO(name, lock)    SEQCOUNT_LOCKNAME_ZERO(name, lock)
#define SEQCNT_SPINLOCK_ZERO(name, lock)    SEQCOUNT_LOCKNAME_ZERO(name, lock)
#define SEQCNT_RWLOCK_ZERO(name, lock)      SEQCOUNT_LOCKNAME_ZERO(name, lock)
#define SEQCNT_MUTEX_ZERO(name, lock)       SEQCOUNT_LOCKNAME_ZERO(name, lock)
#define SEQCNT_WW_MUTEX_ZERO(name, lock)    SEQCOUNT_LOCKNAME_ZERO(name, lock)
 
#define __seqprop_case(s, lockname, prop)               \
    seqcount_##lockname##_t: __seqprop_##lockname##_##prop
 
#define __seqprop(s, prop) _Generic(*(s),               \
    seqcount_t:     __seqprop_##prop,           \
    __seqprop_case((s), raw_spinlock,   prop),          \
    __seqprop_case((s), spinlock,   prop),          \
    __seqprop_case((s), rwlock,     prop),          \
    __seqprop_case((s), mutex,      prop))
 
#define seqprop_ptr(s)          __seqprop(s, ptr)(s)
#define seqprop_const_ptr(s)        __seqprop(s, const_ptr)(s)
#define seqprop_sequence(s)     __seqprop(s, sequence)(s)
#define seqprop_preemptible(s)      __seqprop(s, preemptible)(s)
#define seqprop_assert(s)       __seqprop(s, assert)(s)
 
#define __read_seqcount_begin(s)                    \
({                                  \
    unsigned __seq;                         \
                                    \
    while (unlikely((__seq = seqprop_sequence(s)) & 1))     \
        cpu_relax();                        \
                                    \
    kcsan_atomic_next(KCSAN_SEQLOCK_REGION_MAX);            \
    __seq;                              \
})
 
#define raw_read_seqcount_begin(s) __read_seqcount_begin(s)
 
#define read_seqcount_begin(s)                      \
({                                  \
    seqcount_lockdep_reader_access(seqprop_const_ptr(s));       \
    raw_read_seqcount_begin(s);                 \
})
 
#define raw_read_seqcount(s)                        \
({                                  \
    unsigned __seq = seqprop_sequence(s);               \
                                    \
    kcsan_atomic_next(KCSAN_SEQLOCK_REGION_MAX);            \
    __seq;                              \
})
 
#define raw_seqcount_try_begin(s, start)                \
({                                  \
    start = raw_read_seqcount(s);                   \
    !(start & 1);                           \
})
 
#define raw_seqcount_begin(s)                       \
({                                  \
    /*                              \
     * If the counter is odd, let read_seqcount_retry() fail    \
     * by decrementing the counter.                 \
     */                             \
    raw_read_seqcount(s) & ~1;                  \
})
 
#define __read_seqcount_retry(s, start)                 \
    do___read_seqcount_retry(seqprop_const_ptr(s), start)
 
static inline int do___read_seqcount_retry(const seqcount_t *s, unsigned start)
{
    kcsan_atomic_next(0);
    return unlikely(READ_ONCE(s->sequence) != start);
}
 
#define read_seqcount_retry(s, start)                   \
    do_read_seqcount_retry(seqprop_const_ptr(s), start)
 
static inline int do_read_seqcount_retry(const seqcount_t *s, unsigned start)
{
    smp_rmb();
    return do___read_seqcount_retry(s, start);
}
 
#define raw_write_seqcount_begin(s)                 \
do {                                    \
    if (seqprop_preemptible(s))                 \
        preempt_disable();                  \
                                    \
    do_raw_write_seqcount_begin(seqprop_ptr(s));            \
} while (0)
 
static inline void do_raw_write_seqcount_begin(seqcount_t *s)
{
    kcsan_nestable_atomic_begin();
    s->sequence++;
    smp_wmb();
}
 
#define raw_write_seqcount_end(s)                   \
do {                                    \
    do_raw_write_seqcount_end(seqprop_ptr(s));          \
                                    \
    if (seqprop_preemptible(s))                 \
        preempt_enable();                   \
} while (0)
 
static inline void do_raw_write_seqcount_end(seqcount_t *s)
{
    smp_wmb();
    s->sequence++;
    kcsan_nestable_atomic_end();
}
 
#define write_seqcount_begin_nested(s, subclass)            \
do {                                    \
    seqprop_assert(s);                      \
                                    \
    if (seqprop_preemptible(s))                 \
        preempt_disable();                  \
                                    \
    do_write_seqcount_begin_nested(seqprop_ptr(s), subclass);   \
} while (0)
 
static inline void do_write_seqcount_begin_nested(seqcount_t *s, int subclass)
{
    seqcount_acquire(&s->dep_map, subclass, 0, _RET_IP_);
    do_raw_write_seqcount_begin(s);
}
 
#define write_seqcount_begin(s)                     \
do {                                    \
    seqprop_assert(s);                      \
                                    \
    if (seqprop_preemptible(s))                 \
        preempt_disable();                  \
                                    \
    do_write_seqcount_begin(seqprop_ptr(s));            \
} while (0)
 
static inline void do_write_seqcount_begin(seqcount_t *s)
{
    do_write_seqcount_begin_nested(s, 0);
}
 
#define write_seqcount_end(s)                       \
do {                                    \
    do_write_seqcount_end(seqprop_ptr(s));              \
                                    \
    if (seqprop_preemptible(s))                 \
        preempt_enable();                   \
} while (0)
 
static inline void do_write_seqcount_end(seqcount_t *s)
{
    seqcount_release(&s->dep_map, _RET_IP_);
    do_raw_write_seqcount_end(s);
}
 
#define raw_write_seqcount_barrier(s)                   \
    do_raw_write_seqcount_barrier(seqprop_ptr(s))
 
static inline void do_raw_write_seqcount_barrier(seqcount_t *s)
{
    kcsan_nestable_atomic_begin();
    s->sequence++;
    smp_wmb();
    s->sequence++;
    kcsan_nestable_atomic_end();
}
 
#define write_seqcount_invalidate(s)                    \
    do_write_seqcount_invalidate(seqprop_ptr(s))
 
static inline void do_write_seqcount_invalidate(seqcount_t *s)
{
    smp_wmb();
    kcsan_nestable_atomic_begin();
    s->sequence+=2;
    kcsan_nestable_atomic_end();
}
 
/*
 * Latch sequence counters (seqcount_latch_t)
 *
 * A sequence counter variant where the counter even/odd value is used to
 * switch between two copies of protected data. This allows the read path,
 * typically NMIs, to safely interrupt the write side critical section.
 *
 * As the write sections are fully preemptible, no special handling for
 * PREEMPT_RT is needed.
 */
typedef struct {
    seqcount_t seqcount;
} seqcount_latch_t;
 
#define SEQCNT_LATCH_ZERO(seq_name) {                   \
    .seqcount       = SEQCNT_ZERO(seq_name.seqcount),   \
}
 
#define seqcount_latch_init(s) seqcount_init(&(s)->seqcount)
 
static __always_inline unsigned raw_read_seqcount_latch(const seqcount_latch_t *s)
{
    /*
     * Pairs with the first smp_wmb() in raw_write_seqcount_latch().
     * Due to the dependent load, a full smp_rmb() is not needed.
     */
    return READ_ONCE(s->seqcount.sequence);
}
 
static __always_inline unsigned read_seqcount_latch(const seqcount_latch_t *s)
{
    kcsan_atomic_next(KCSAN_SEQLOCK_REGION_MAX);
    return raw_read_seqcount_latch(s);
}
 
static __always_inline int
raw_read_seqcount_latch_retry(const seqcount_latch_t *s, unsigned start)
{
    smp_rmb();
    return unlikely(READ_ONCE(s->seqcount.sequence) != start);
}
 
static __always_inline int
read_seqcount_latch_retry(const seqcount_latch_t *s, unsigned start)
{
    kcsan_atomic_next(0);
    return raw_read_seqcount_latch_retry(s, start);
}
 
static __always_inline void raw_write_seqcount_latch(seqcount_latch_t *s)
{
    smp_wmb();  /* prior stores before incrementing "sequence" */
    s->seqcount.sequence++;
    smp_wmb();      /* increment "sequence" before following stores */
}
 
static __always_inline void write_seqcount_latch_begin(seqcount_latch_t *s)
{
    kcsan_nestable_atomic_begin();
    raw_write_seqcount_latch(s);
}
 
static __always_inline void write_seqcount_latch(seqcount_latch_t *s)
{
    raw_write_seqcount_latch(s);
}
 
static __always_inline void write_seqcount_latch_end(seqcount_latch_t *s)
{
    kcsan_nestable_atomic_end();
}
 
#define __SEQLOCK_UNLOCKED(lockname)                    \
    {                               \
        .seqcount = SEQCNT_SPINLOCK_ZERO(lockname, &(lockname).lock), \
        .lock = __SPIN_LOCK_UNLOCKED(lockname)          \
    }
 
#define seqlock_init(sl)                        \
    do {                                \
        spin_lock_init(&(sl)->lock);                \
        seqcount_spinlock_init(&(sl)->seqcount, &(sl)->lock);   \
    } while (0)
 
#define DEFINE_SEQLOCK(sl) \
        seqlock_t sl = __SEQLOCK_UNLOCKED(sl)
 
static inline unsigned read_seqbegin(const seqlock_t *sl)
    __acquires_shared(sl) __no_context_analysis
{
    return read_seqcount_begin(&sl->seqcount);
}
 
static inline unsigned read_seqretry(const seqlock_t *sl, unsigned start)
    __releases_shared(sl) __no_context_analysis
{
    return read_seqcount_retry(&sl->seqcount, start);
}
 
/*
 * For all seqlock_t write side functions, use the internal
 * do_write_seqcount_begin() instead of generic write_seqcount_begin().
 * This way, no redundant lockdep_assert_held() checks are added.
 */
 
static inline void write_seqlock(seqlock_t *sl)
    __acquires(sl) __no_context_analysis
{
    spin_lock(&sl->lock);
    do_write_seqcount_begin(&sl->seqcount.seqcount);
}
 
static inline void write_sequnlock(seqlock_t *sl)
    __releases(sl) __no_context_analysis
{
    do_write_seqcount_end(&sl->seqcount.seqcount);
    spin_unlock(&sl->lock);
}
 
static inline void write_seqlock_bh(seqlock_t *sl)
    __acquires(sl) __no_context_analysis
{
    spin_lock_bh(&sl->lock);
    do_write_seqcount_begin(&sl->seqcount.seqcount);
}
 
static inline void write_sequnlock_bh(seqlock_t *sl)
    __releases(sl) __no_context_analysis
{
    do_write_seqcount_end(&sl->seqcount.seqcount);
    spin_unlock_bh(&sl->lock);
}
 
static inline void write_seqlock_irq(seqlock_t *sl)
    __acquires(sl) __no_context_analysis
{
    spin_lock_irq(&sl->lock);
    do_write_seqcount_begin(&sl->seqcount.seqcount);
}
 
static inline void write_sequnlock_irq(seqlock_t *sl)
    __releases(sl) __no_context_analysis
{
    do_write_seqcount_end(&sl->seqcount.seqcount);
    spin_unlock_irq(&sl->lock);
}
 
static inline unsigned long __write_seqlock_irqsave(seqlock_t *sl)
    __acquires(sl) __no_context_analysis
{
    unsigned long flags;
 
    spin_lock_irqsave(&sl->lock, flags);
    do_write_seqcount_begin(&sl->seqcount.seqcount);
    return flags;
}
 
#define write_seqlock_irqsave(lock, flags)              \
    do { flags = __write_seqlock_irqsave(lock); } while (0)
 
static inline void
write_sequnlock_irqrestore(seqlock_t *sl, unsigned long flags)
    __releases(sl) __no_context_analysis
{
    do_write_seqcount_end(&sl->seqcount.seqcount);
    spin_unlock_irqrestore(&sl->lock, flags);
}
 
static inline void read_seqlock_excl(seqlock_t *sl)
    __acquires_shared(sl) __no_context_analysis
{
    spin_lock(&sl->lock);
}
 
static inline void read_sequnlock_excl(seqlock_t *sl)
    __releases_shared(sl) __no_context_analysis
{
    spin_unlock(&sl->lock);
}
 
static inline void read_seqlock_excl_bh(seqlock_t *sl)
    __acquires_shared(sl) __no_context_analysis
{
    spin_lock_bh(&sl->lock);
}
 
static inline void read_sequnlock_excl_bh(seqlock_t *sl)
    __releases_shared(sl) __no_context_analysis
{
    spin_unlock_bh(&sl->lock);
}
 
static inline void read_seqlock_excl_irq(seqlock_t *sl)
    __acquires_shared(sl) __no_context_analysis
{
    spin_lock_irq(&sl->lock);
}
 
static inline void read_sequnlock_excl_irq(seqlock_t *sl)
    __releases_shared(sl) __no_context_analysis
{
    spin_unlock_irq(&sl->lock);
}
 
static inline unsigned long __read_seqlock_excl_irqsave(seqlock_t *sl)
    __acquires_shared(sl) __no_context_analysis
{
    unsigned long flags;
 
    spin_lock_irqsave(&sl->lock, flags);
    return flags;
}
 
#define read_seqlock_excl_irqsave(lock, flags)              \
    do { flags = __read_seqlock_excl_irqsave(lock); } while (0)
 
static inline void
read_sequnlock_excl_irqrestore(seqlock_t *sl, unsigned long flags)
    __releases_shared(sl) __no_context_analysis
{
    spin_unlock_irqrestore(&sl->lock, flags);
}
 
static inline void read_seqbegin_or_lock(seqlock_t *lock, int *seq)
    __acquires_shared(lock) __no_context_analysis
{
    if (!(*seq & 1))    /* Even */
        *seq = read_seqbegin(lock);
    else            /* Odd */
        read_seqlock_excl(lock);
}
 
static inline int need_seqretry(seqlock_t *lock, int seq)
    __releases_shared(lock) __no_context_analysis
{
    return !(seq & 1) && read_seqretry(lock, seq);
}
 
static inline void done_seqretry(seqlock_t *lock, int seq)
    __no_context_analysis
{
    if (seq & 1)
        read_sequnlock_excl(lock);
}
 
static inline unsigned long
read_seqbegin_or_lock_irqsave(seqlock_t *lock, int *seq)
    __acquires_shared(lock) __no_context_analysis
{
    unsigned long flags = 0;
 
    if (!(*seq & 1))    /* Even */
        *seq = read_seqbegin(lock);
    else            /* Odd */
        read_seqlock_excl_irqsave(lock, flags);
 
    return flags;
}
 
static inline void
done_seqretry_irqrestore(seqlock_t *lock, int seq, unsigned long flags)
    __no_context_analysis
{
    if (seq & 1)
        read_sequnlock_excl_irqrestore(lock, flags);
}
 
enum ss_state {
    ss_done = 0,
    ss_lock,
    ss_lock_irqsave,
    ss_lockless,
};
 
struct ss_tmp {
    enum ss_state   state;
    unsigned long   data;
    spinlock_t  *lock;
    spinlock_t  *lock_irqsave;
};
 
static __always_inline void __scoped_seqlock_cleanup(struct ss_tmp *sst)
    __no_context_analysis
{
    if (sst->lock)
        spin_unlock(sst->lock);
    if (sst->lock_irqsave)
        spin_unlock_irqrestore(sst->lock_irqsave, sst->data);
}
 
extern void __scoped_seqlock_invalid_target(void);
 
#if (defined(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION < 90000) || \
    defined(CONFIG_KASAN) || defined(CONFIG_UBSAN_ALIGNMENT)
/*
 * For some reason some GCC-8 architectures (nios2, alpha) have trouble
 * determining that the ss_done state is impossible in __scoped_seqlock_next()
 * below.
 *
 * Similarly KASAN and UBSAN_ALIGNMENT are known to confuse compilers enough
 * to break this. But we don't care about code quality for such builds anyway.
 */
static inline void __scoped_seqlock_bug(void) { }
#else
/*
 * Canary for compiler optimization -- if the compiler doesn't realize this is
 * an impossible state, it very likely generates sub-optimal code here.
 */
extern void __scoped_seqlock_bug(void);
#endif
 
static __always_inline void
__scoped_seqlock_next(struct ss_tmp *sst, seqlock_t *lock, enum ss_state target)
    __no_context_analysis
{
    switch (sst->state) {
    case ss_done:
        __scoped_seqlock_bug();
        return;
 
    case ss_lock:
    case ss_lock_irqsave:
        sst->state = ss_done;
        return;
 
    case ss_lockless:
        if (!read_seqretry(lock, sst->data)) {
            sst->state = ss_done;
            return;
        }
        break;
    }
 
    switch (target) {
    case ss_done:
        __scoped_seqlock_invalid_target();
        return;
 
    case ss_lock:
        sst->lock = &lock->lock;
        spin_lock(sst->lock);
        sst->state = ss_lock;
        return;
 
    case ss_lock_irqsave:
        sst->lock_irqsave = &lock->lock;
        spin_lock_irqsave(sst->lock_irqsave, sst->data);
        sst->state = ss_lock_irqsave;
        return;
 
    case ss_lockless:
        sst->data = read_seqbegin(lock);
        return;
    }
}
 
/*
 * Context analysis no-op helper to release seqlock at the end of the for-scope;
 * the alias analysis of the compiler will recognize that the pointer @s is an
 * alias to @_seqlock passed to read_seqbegin(_seqlock) below.
 */
static __always_inline void __scoped_seqlock_cleanup_ctx(struct ss_tmp **s)
    __releases_shared(*((seqlock_t **)s)) __no_context_analysis {}
 
#define __scoped_seqlock_read(_seqlock, _target, _s)            \
    for (struct ss_tmp _s __cleanup(__scoped_seqlock_cleanup) = \
         { .state = ss_lockless, .data = read_seqbegin(_seqlock) }, \
         *__UNIQUE_ID(ctx) __cleanup(__scoped_seqlock_cleanup_ctx) =\
        (struct ss_tmp *)_seqlock;              \
         _s.state != ss_done;                   \
         __scoped_seqlock_next(&_s, _seqlock, _target))
 
#define scoped_seqlock_read(_seqlock, _target)              \
    __scoped_seqlock_read(_seqlock, _target, __UNIQUE_ID(seqlock))
 
DEFINE_LOCK_GUARD_1(seqlock_init, seqlock_t, seqlock_init(_T->lock), /* */)
DECLARE_LOCK_GUARD_1_ATTRS(seqlock_init, __acquires(_T), __releases(*(seqlock_t **)_T))
#define class_seqlock_init_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(seqlock_init, _T)
 
#endif /* __LINUX_SEQLOCK_H */