seqlock.h File Reference

#include <linux/compiler.h>
#include <linux/cleanup.h>
#include <linux/kcsan-checks.h>
#include <linux/lockdep.h>
#include <linux/mutex.h>
#include <linux/preempt.h>
#include <linux/seqlock_types.h>
#include <linux/spinlock.h>
#include <asm/processor.h>

Include dependency graph for seqlock.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Classes
struct	seqcount_latch_t
struct	ss_tmp

Macros
#define	KCSAN_SEQLOCK_REGION_MAX   1000
#define	SEQCOUNT_DEP_MAP_INIT(lockname)
#define	seqcount_init(s)   __seqcount_init(s, NULL, NULL)
#define	seqcount_lockdep_reader_access(x)
#define	class_seqlock_init_constructor(_T)   WITH_LOCK_GUARD_1_ATTRS(seqlock_init, _T)

: Name of the seqcount_t instance
SEQCNT_ZERO() - static initializer for seqcount_t
#define	SEQCNT_ZERO(name)   { .sequence = 0, SEQCOUNT_DEP_MAP_INIT(name) }
#define	seqcount_LOCKNAME_init(s, _lock, lockname)
#define	seqcount_raw_spinlock_init(s, lock)   seqcount_LOCKNAME_init(s, lock, raw_spinlock)
#define	seqcount_spinlock_init(s, lock)   seqcount_LOCKNAME_init(s, lock, spinlock)
#define	seqcount_rwlock_init(s, lock)   seqcount_LOCKNAME_init(s, lock, rwlock)
#define	seqcount_mutex_init(s, lock)   seqcount_LOCKNAME_init(s, lock, mutex)
#define	SEQCOUNT_LOCKNAME(lockname, locktype, preemptible, lockbase)
#define	__SEQ_RT   IS_ENABLED(CONFIG_PREEMPT_RT)
#define	SEQCOUNT_LOCKNAME_ZERO(seq_name, assoc_lock)
#define	SEQCNT_RAW_SPINLOCK_ZERO(name, lock)   SEQCOUNT_LOCKNAME_ZERO(name, lock)
#define	SEQCNT_SPINLOCK_ZERO(name, lock)   SEQCOUNT_LOCKNAME_ZERO(name, lock)
#define	SEQCNT_RWLOCK_ZERO(name, lock)   SEQCOUNT_LOCKNAME_ZERO(name, lock)
#define	SEQCNT_MUTEX_ZERO(name, lock)   SEQCOUNT_LOCKNAME_ZERO(name, lock)
#define	SEQCNT_WW_MUTEX_ZERO(name, lock)   SEQCOUNT_LOCKNAME_ZERO(name, lock)
#define	__seqprop_case(s, lockname, prop)    seqcount_##lockname##_t: __seqprop_##lockname##_##prop
#define	__seqprop(s, prop)
#define	seqprop_ptr(s)   __seqprop(s, ptr)(s)
#define	seqprop_const_ptr(s)   __seqprop(s, const_ptr)(s)
#define	seqprop_sequence(s)   __seqprop(s, sequence)(s)
#define	seqprop_preemptible(s)   __seqprop(s, preemptible)(s)
#define	seqprop_assert(s)   __seqprop(s, assert)(s)
#define	__read_seqcount_begin(s)
#define	raw_read_seqcount_begin(s)   __read_seqcount_begin(s)
#define	read_seqcount_begin(s)
#define	raw_read_seqcount(s)
#define	raw_seqcount_try_begin(s, start)
#define	raw_seqcount_begin(s)
#define	__read_seqcount_retry(s, start)    do___read_seqcount_retry(seqprop_const_ptr(s), start)
#define	read_seqcount_retry(s, start)    do_read_seqcount_retry(seqprop_const_ptr(s), start)
#define	raw_write_seqcount_begin(s)
#define	raw_write_seqcount_end(s)
#define	write_seqcount_begin_nested(s, subclass)
#define	write_seqcount_begin(s)
#define	write_seqcount_end(s)
#define	raw_write_seqcount_barrier(s)    do_raw_write_seqcount_barrier(seqprop_ptr(s))
#define	write_seqcount_invalidate(s)    do_write_seqcount_invalidate(seqprop_ptr(s))
#define	SEQCNT_LATCH_ZERO(seq_name)
#define	seqcount_latch_init(s)   seqcount_init(&(s)->seqcount)
#define	__SEQLOCK_UNLOCKED(lockname)
#define	seqlock_init(sl)
#define	DEFINE_SEQLOCK(sl)    seqlock_t sl = __SEQLOCK_UNLOCKED(sl)
#define	write_seqlock_irqsave(lock, flags)    do { flags = __write_seqlock_irqsave(lock); } while (0)
#define	read_seqlock_excl_irqsave(lock, flags)    do { flags = __read_seqlock_excl_irqsave(lock); } while (0)
#define	__scoped_seqlock_read(_seqlock, _target, _s)
#define	scoped_seqlock_read(_seqlock, _target)    __scoped_seqlock_read(_seqlock, _target, __UNIQUE_ID(seqlock))
enum	ss_state { ss_done = 0 , ss_lock , ss_lock_irqsave , ss_lockless }
void	__scoped_seqlock_invalid_target (void)
void	__scoped_seqlock_bug (void)

Macro Definition Documentation

__read_seqcount_begin

#define __read_seqcount_begin

(

s

)

Value:

({                                  \
    unsigned __seq;                         \
                                    \
    while (unlikely((__seq = seqprop_sequence(s)) & 1))     \
        cpu_relax();                        \
                                    \
    kcsan_atomic_next(KCSAN_SEQLOCK_REGION_MAX);            \
    __seq;                              \
})

__read_seqcount_begin() - begin a seqcount_t read section @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants

Return: count to be passed to read_seqcount_retry()

__read_seqcount_retry

#define __read_seqcount_retry	(		s,
			start
	)		do___read_seqcount_retry(seqprop_const_ptr(s), start)

__read_seqcount_retry() - end a seqcount_t read section w/o barrier @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants @start: count, from read_seqcount_begin()

__read_seqcount_retry is like read_seqcount_retry, but has no smp_rmb() barrier. Callers should ensure that smp_rmb() or equivalent ordering is provided before actually loading any of the variables that are to be protected in this critical section.

Use carefully, only in critical code, and comment how the barrier is provided.

Return: true if a read section retry is required, else false

__scoped_seqlock_read

#define __scoped_seqlock_read	(		_seqlock,
			_target,
			_s
	)

Value:

    for (struct ss_tmp _s __cleanup(__scoped_seqlock_cleanup) = \
         { .state = ss_lockless, .data = read_seqbegin(_seqlock) }, \
         *__UNIQUE_ID(ctx) __cleanup(__scoped_seqlock_cleanup_ctx) =\
        (struct ss_tmp *)_seqlock;              \
         _s.state != ss_done;                   \
         __scoped_seqlock_next(&_s, _seqlock, _target))

__SEQ_RT

#define __SEQ_RT   IS_ENABLED(CONFIG_PREEMPT_RT)

__SEQLOCK_UNLOCKED

#define __SEQLOCK_UNLOCKED

(

lockname

)

Value:

    {                               \
        .seqcount = SEQCNT_SPINLOCK_ZERO(lockname, &(lockname).lock), \
        .lock = __SPIN_LOCK_UNLOCKED(lockname)          \
    }

__seqprop

#define __seqprop	(		s,
			prop
	)

Value:

    _Generic(*(s),              \
    seqcount_t:     __seqprop_##prop,           \
    __seqprop_case((s), raw_spinlock,   prop),          \
    __seqprop_case((s), spinlock,   prop),          \
    __seqprop_case((s), rwlock,     prop),          \
    __seqprop_case((s), mutex,      prop))

__seqprop_case

#define __seqprop_case	(		s,
			lockname,
			prop
	)		seqcount_##lockname##_t: __seqprop_##lockname##_##prop

class_seqlock_init_constructor

#define class_seqlock_init_constructor

(

_T

)

WITH_LOCK_GUARD_1_ATTRS(seqlock_init, _T)

DEFINE_SEQLOCK

#define DEFINE_SEQLOCK

(

sl

)

seqlock_t sl = __SEQLOCK_UNLOCKED(sl)

DEFINE_SEQLOCK(sl) - Define a statically allocated seqlock_t @sl: Name of the seqlock_t instance

KCSAN_SEQLOCK_REGION_MAX

#define KCSAN_SEQLOCK_REGION_MAX   1000

raw_read_seqcount

#define raw_read_seqcount

(

s

)

Value:

({                                  \
    unsigned __seq = seqprop_sequence(s);               \
                                    \
    kcsan_atomic_next(KCSAN_SEQLOCK_REGION_MAX);            \
    __seq;                              \
})

raw_read_seqcount() - read the raw seqcount_t counter value @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants

raw_read_seqcount opens a read critical section of the given seqcount_t, without any lockdep checking, and without checking or masking the sequence counter LSB. Calling code is responsible for handling that.

Return: count to be passed to read_seqcount_retry()

raw_read_seqcount_begin

#define raw_read_seqcount_begin

(

s

)

__read_seqcount_begin(s)

raw_read_seqcount_begin() - begin a seqcount_t read section w/o lockdep @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants

Return: count to be passed to read_seqcount_retry()

raw_seqcount_begin

#define raw_seqcount_begin

(

s

)

Value:

({                                  \
    /*                              \
     * If the counter is odd, let read_seqcount_retry() fail    \
     * by decrementing the counter.                 \
     */                             \
    raw_read_seqcount(s) & ~1;                  \
})

raw_seqcount_begin() - begin a seqcount_t read critical section w/o lockdep and w/o counter stabilization @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants

raw_seqcount_begin opens a read critical section of the given seqcount_t. Unlike read_seqcount_begin(), this function will not wait for the count to stabilize. If a writer is active when it begins, it will fail the read_seqcount_retry() at the end of the read critical section instead of stabilizing at the beginning of it.

Use this only in special kernel hot paths where the read section is small and has a high probability of success through other external means. It will save a single branching instruction.

Return: count to be passed to read_seqcount_retry()

raw_seqcount_try_begin

#define raw_seqcount_try_begin	(		s,
			start
	)

Value:

({                                  \
    start = raw_read_seqcount(s);                   \
    !(start & 1);                           \
})

raw_seqcount_try_begin() - begin a seqcount_t read critical section w/o lockdep and w/o counter stabilization @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants @start: count to be passed to read_seqcount_retry()

Similar to raw_seqcount_begin(), except it enables eliding the critical section entirely if odd, instead of doing the speculation knowing it will fail.

Useful when counter stabilization is more or less equivalent to taking the lock and there is a slowpath that does that.

If true, start will be set to the (even) sequence count read.

Return: true when a read critical section is started.

raw_write_seqcount_barrier

#define raw_write_seqcount_barrier

(

s

)

do_raw_write_seqcount_barrier(seqprop_ptr(s))

raw_write_seqcount_barrier() - do a seqcount_t write barrier @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants

This can be used to provide an ordering guarantee instead of the usual consistency guarantee. It is one wmb cheaper, because it can collapse the two back-to-back wmb()s.

Note that writes surrounding the barrier should be declared atomic (e.g. via WRITE_ONCE): a) to ensure the writes become visible to other threads atomically, avoiding compiler optimizations; b) to document which writes are meant to propagate to the reader critical section. This is necessary because neither writes before nor after the barrier are enclosed in a seq-writer critical section that would ensure readers are aware of ongoing writes::

seqcount_t seq; bool X = true, Y = false;

void read(void) { bool x, y;

do { int s = read_seqcount_begin(&seq);

x = X; y = Y;

} while (read_seqcount_retry(&seq, s));

BUG_ON(!x && !y); }

void write(void) { WRITE_ONCE(Y, true);

raw_write_seqcount_barrier(seq);

WRITE_ONCE(X, false); }

raw_write_seqcount_begin

#define raw_write_seqcount_begin

(

s

)

Value:

do {                                    \
    if (seqprop_preemptible(s))                 \
        preempt_disable();                  \
                                    \
    do_raw_write_seqcount_begin(seqprop_ptr(s));            \
} while (0)

raw_write_seqcount_begin() - start a seqcount_t write section w/o lockdep @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants

Context: check write_seqcount_begin()

raw_write_seqcount_end

#define raw_write_seqcount_end

(

s

)

Value:

do {                                    \
    do_raw_write_seqcount_end(seqprop_ptr(s));          \
                                    \
    if (seqprop_preemptible(s))                 \
        preempt_enable();                   \
} while (0)

raw_write_seqcount_end() - end a seqcount_t write section w/o lockdep @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants

Context: check write_seqcount_end()

read_seqcount_begin

#define read_seqcount_begin

(

s

)

Value:

({                                  \
    seqcount_lockdep_reader_access(seqprop_const_ptr(s));       \
    raw_read_seqcount_begin(s);                 \
})

read_seqcount_begin() - begin a seqcount_t read critical section @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants

Return: count to be passed to read_seqcount_retry()

read_seqcount_retry

#define read_seqcount_retry	(		s,
			start
	)		do_read_seqcount_retry(seqprop_const_ptr(s), start)

read_seqcount_retry() - end a seqcount_t read critical section @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants @start: count, from read_seqcount_begin()

read_seqcount_retry closes the read critical section of given seqcount_t. If the critical section was invalid, it must be ignored (and typically retried).

Return: true if a read section retry is required, else false

read_seqlock_excl_irqsave

#define read_seqlock_excl_irqsave	(		lock,
			flags
	)		do { flags = __read_seqlock_excl_irqsave(lock); } while (0)

read_seqlock_excl_irqsave() - start a non-interruptible seqlock_t locking reader section @lock: Pointer to seqlock_t @flags: Stack-allocated storage for saving caller's local interrupt state, to be passed to read_sequnlock_excl_irqrestore().

_irqsave variant of read_seqlock_excl(). Use this only if the seqlock_t write side section, or other read sections, can be invoked from a hardirq context.

scoped_seqlock_read

#define scoped_seqlock_read	(		_seqlock,
			_target
	)		__scoped_seqlock_read(_seqlock, _target, __UNIQUE_ID(seqlock))

scoped_seqlock_read() - execute the read-side critical section without manual sequence counter handling or calls to other helpers @_seqlock: pointer to seqlock_t protecting the data @_target: an enum ss_state: one of {ss_lock, ss_lock_irqsave, ss_lockless} indicating the type of critical read section

Example::

scoped_seqlock_read (&lock, ss_lock) {
    // read-side critical section
}

Starts with a lockess pass first. If it fails, restarts the critical section with the lock held.

SEQCNT_LATCH_ZERO

#define SEQCNT_LATCH_ZERO

(

seq_name

)

Value:

    {                   \
    .seqcount       = SEQCNT_ZERO(seq_name.seqcount),   \
}

SEQCNT_LATCH_ZERO() - static initializer for seqcount_latch_t @seq_name: Name of the seqcount_latch_t instance

SEQCNT_MUTEX_ZERO

#define SEQCNT_MUTEX_ZERO	(		name,
			lock
	)		SEQCOUNT_LOCKNAME_ZERO(name, lock)

SEQCNT_RAW_SPINLOCK_ZERO

#define SEQCNT_RAW_SPINLOCK_ZERO	(		name,
			lock
	)		SEQCOUNT_LOCKNAME_ZERO(name, lock)

SEQCNT_RWLOCK_ZERO

#define SEQCNT_RWLOCK_ZERO	(		name,
			lock
	)		SEQCOUNT_LOCKNAME_ZERO(name, lock)

SEQCNT_SPINLOCK_ZERO

#define SEQCNT_SPINLOCK_ZERO	(		name,
			lock
	)		SEQCOUNT_LOCKNAME_ZERO(name, lock)

SEQCNT_WW_MUTEX_ZERO

#define SEQCNT_WW_MUTEX_ZERO	(		name,
			lock
	)		SEQCOUNT_LOCKNAME_ZERO(name, lock)

SEQCNT_ZERO

#define SEQCNT_ZERO

(

name

)

{ .sequence = 0, SEQCOUNT_DEP_MAP_INIT(name) }

SEQCOUNT_DEP_MAP_INIT

#define SEQCOUNT_DEP_MAP_INIT

(

lockname

)

seqcount_init

#define seqcount_init

(

s

)

__seqcount_init(s, NULL, NULL)

seqcount_latch_init

#define seqcount_latch_init

(

s

)

seqcount_init(&(s)->seqcount)

seqcount_latch_init() - runtime initializer for seqcount_latch_t @s: Pointer to the seqcount_latch_t instance

seqcount_lockdep_reader_access

#define seqcount_lockdep_reader_access

(

x

)

SEQCOUNT_LOCKNAME

#define SEQCOUNT_LOCKNAME	(		lockname,
			locktype,
			preemptible,
			lockbase
	)

seqcount_LOCKNAME_init

#define seqcount_LOCKNAME_init	(		s,
			_lock,
			lockname
	)

Value:

    do {                                \
        seqcount_##lockname##_t *____s = (s);           \
        seqcount_init(&____s->seqcount);            \
        __SEQ_LOCK(____s->lock = (_lock));          \
    } while (0)

SEQCOUNT_LOCKNAME_ZERO

#define SEQCOUNT_LOCKNAME_ZERO	(		seq_name,
			assoc_lock
	)

Value:

    {           \
    .seqcount       = SEQCNT_ZERO(seq_name.seqcount),   \
    __SEQ_LOCK(.lock    = (assoc_lock))             \
}

seqcount_mutex_init

#define seqcount_mutex_init	(		s,
			lock
	)		seqcount_LOCKNAME_init(s, lock, mutex)

seqcount_raw_spinlock_init

#define seqcount_raw_spinlock_init	(		s,
			lock
	)		seqcount_LOCKNAME_init(s, lock, raw_spinlock)

seqcount_rwlock_init

#define seqcount_rwlock_init	(		s,
			lock
	)		seqcount_LOCKNAME_init(s, lock, rwlock)

seqcount_spinlock_init

#define seqcount_spinlock_init	(		s,
			lock
	)		seqcount_LOCKNAME_init(s, lock, spinlock)

seqlock_init

#define seqlock_init

(

sl

)

Value:

    do {                                \
        spin_lock_init(&(sl)->lock);                \
        seqcount_spinlock_init(&(sl)->seqcount, &(sl)->lock);   \
    } while (0)

seqlock_init() - dynamic initializer for seqlock_t @sl: Pointer to the seqlock_t instance

seqprop_assert

#define seqprop_assert

(

s

)

__seqprop(s, assert)(s)

seqprop_const_ptr

#define seqprop_const_ptr

(

s

)

__seqprop(s, const_ptr)(s)

seqprop_preemptible

#define seqprop_preemptible

(

s

)

__seqprop(s, preemptible)(s)

seqprop_ptr

#define seqprop_ptr

(

s

)

__seqprop(s, ptr)(s)

seqprop_sequence

#define seqprop_sequence

(

s

)

__seqprop(s, sequence)(s)

write_seqcount_begin

#define write_seqcount_begin

(

s

)

Value:

do {                                    \
    seqprop_assert(s);                      \
                                    \
    if (seqprop_preemptible(s))                 \
        preempt_disable();                  \
                                    \
    do_write_seqcount_begin(seqprop_ptr(s));            \
} while (0)

write_seqcount_begin() - start a seqcount_t write side critical section @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants

Context: sequence counter write side sections must be serialized and non-preemptible. Preemption will be automatically disabled if and only if the seqcount write serialization lock is associated, and preemptible. If readers can be invoked from hardirq or softirq context, interrupts or bottom halves must be respectively disabled.

write_seqcount_begin_nested

#define write_seqcount_begin_nested	(		s,
			subclass
	)

Value:

do {                                    \
    seqprop_assert(s);                      \
                                    \
    if (seqprop_preemptible(s))                 \
        preempt_disable();                  \
                                    \
    do_write_seqcount_begin_nested(seqprop_ptr(s), subclass);   \
} while (0)

write_seqcount_begin_nested() - start a seqcount_t write section with custom lockdep nesting level @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants @subclass: lockdep nesting level

See Documentation/locking/lockdep-design.rst Context: check write_seqcount_begin()

write_seqcount_end

#define write_seqcount_end

(

s

)

Value:

do {                                    \
    do_write_seqcount_end(seqprop_ptr(s));              \
                                    \
    if (seqprop_preemptible(s))                 \
        preempt_enable();                   \
} while (0)

write_seqcount_end() - end a seqcount_t write side critical section @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants

Context: Preemption will be automatically re-enabled if and only if the seqcount write serialization lock is associated, and preemptible.

write_seqcount_invalidate

#define write_seqcount_invalidate

(

s

)

do_write_seqcount_invalidate(seqprop_ptr(s))

write_seqcount_invalidate() - invalidate in-progress seqcount_t read side operations @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants

After write_seqcount_invalidate, no seqcount_t read side operations will complete successfully and see data older than this.

write_seqlock_irqsave

#define write_seqlock_irqsave	(		lock,
			flags
	)		do { flags = __write_seqlock_irqsave(lock); } while (0)

write_seqlock_irqsave() - start a non-interruptible seqlock_t write section @lock: Pointer to seqlock_t @flags: Stack-allocated storage for saving caller's local interrupt state, to be passed to write_sequnlock_irqrestore().

_irqsave variant of write_seqlock(). Use it only if the read side section, or other write sections, can be invoked from hardirq context.

Enumeration Type Documentation

ss_state

enum ss_state

Enumerator
ss_done
ss_lock
ss_lock_irqsave
ss_lockless

Function Documentation

__scoped_seqlock_bug()

void __scoped_seqlock_bug ( void  )

extern

__scoped_seqlock_invalid_target()

void __scoped_seqlock_invalid_target ( void  )

extern