kcsan-checks.h

Go to the documentation of this file.

/* SPDX-License-Identifier: GPL-2.0 */
/*
 * KCSAN access checks and modifiers. These can be used to explicitly check
 * uninstrumented accesses, or change KCSAN checking behaviour of accesses.
 *
 * Copyright (C) 2019, Google LLC.
 */
 
#ifndef _LINUX_KCSAN_CHECKS_H
#define _LINUX_KCSAN_CHECKS_H
 
/* Note: Only include what is already included by compiler.h. */
#include <linux/compiler_attributes.h>
#include <linux/types.h>
 
/* Access types -- if KCSAN_ACCESS_WRITE is not set, the access is a read. */
#define KCSAN_ACCESS_WRITE  (1 << 0) /* Access is a write. */
#define KCSAN_ACCESS_COMPOUND   (1 << 1) /* Compounded read-write instrumentation. */
#define KCSAN_ACCESS_ATOMIC (1 << 2) /* Access is atomic. */
/* The following are special, and never due to compiler instrumentation. */
#define KCSAN_ACCESS_ASSERT (1 << 3) /* Access is an assertion. */
#define KCSAN_ACCESS_SCOPED (1 << 4) /* Access is a scoped access. */
 
/*
 * __kcsan_*: Always calls into the runtime when KCSAN is enabled. This may be used
 * even in compilation units that selectively disable KCSAN, but must use KCSAN
 * to validate access to an address. Never use these in header files!
 */
#ifdef CONFIG_KCSAN
void __kcsan_check_access(const volatile void *ptr, size_t size, int type);
 
/*
 * See definition of __tsan_atomic_signal_fence() in kernel/kcsan/core.c.
 * Note: The mappings are arbitrary, and do not reflect any real mappings of C11
 * memory orders to the LKMM memory orders and vice-versa!
 */
#define __KCSAN_BARRIER_TO_SIGNAL_FENCE_mb  __ATOMIC_SEQ_CST
#define __KCSAN_BARRIER_TO_SIGNAL_FENCE_wmb __ATOMIC_ACQ_REL
#define __KCSAN_BARRIER_TO_SIGNAL_FENCE_rmb __ATOMIC_ACQUIRE
#define __KCSAN_BARRIER_TO_SIGNAL_FENCE_release __ATOMIC_RELEASE
 
void __kcsan_mb(void);
 
void __kcsan_wmb(void);
 
void __kcsan_rmb(void);
 
void __kcsan_release(void);
 
void kcsan_disable_current(void);
 
void kcsan_enable_current(void);
void kcsan_enable_current_nowarn(void); /* Safe in uaccess regions. */
 
void kcsan_nestable_atomic_begin(void);
 
void kcsan_nestable_atomic_end(void);
 
void kcsan_flat_atomic_begin(void);
 
void kcsan_flat_atomic_end(void);
 
void kcsan_atomic_next(int n);
 
void kcsan_set_access_mask(unsigned long mask);
 
/* Scoped access information. */
struct kcsan_scoped_access {
    union {
        struct list_head list; /* scoped_accesses list */
        /*
         * Not an entry in scoped_accesses list; stack depth from where
         * the access was initialized.
         */
        int stack_depth;
    };
 
    /* Access information. */
    const volatile void *ptr;
    size_t size;
    int type;
    /* Location where scoped access was set up. */
    unsigned long ip;
};
/*
 * Automatically call kcsan_end_scoped_access() when kcsan_scoped_access goes
 * out of scope; relies on attribute "cleanup", which is supported by all
 * compilers that support KCSAN.
 */
#define __kcsan_cleanup_scoped                                                 \
    __maybe_unused __attribute__((__cleanup__(kcsan_end_scoped_access)))
 
struct kcsan_scoped_access *
kcsan_begin_scoped_access(const volatile void *ptr, size_t size, int type,
              struct kcsan_scoped_access *sa);
 
void kcsan_end_scoped_access(struct kcsan_scoped_access *sa);
 
 
#else /* CONFIG_KCSAN */
 
static inline void __kcsan_check_access(const volatile void *ptr, size_t size,
                    int type) { }
 
static inline void __kcsan_mb(void)         { }
static inline void __kcsan_wmb(void)            { }
static inline void __kcsan_rmb(void)            { }
static inline void __kcsan_release(void)        { }
static inline void kcsan_disable_current(void)      { }
static inline void kcsan_enable_current(void)       { }
static inline void kcsan_enable_current_nowarn(void)    { }
static inline void kcsan_nestable_atomic_begin(void)    { }
static inline void kcsan_nestable_atomic_end(void)  { }
static inline void kcsan_flat_atomic_begin(void)    { }
static inline void kcsan_flat_atomic_end(void)      { }
static inline void kcsan_atomic_next(int n)     { }
static inline void kcsan_set_access_mask(unsigned long mask) { }
 
struct kcsan_scoped_access { };
#define __kcsan_cleanup_scoped __maybe_unused
static inline struct kcsan_scoped_access *
kcsan_begin_scoped_access(const volatile void *ptr, size_t size, int type,
              struct kcsan_scoped_access *sa) { return sa; }
static inline void kcsan_end_scoped_access(struct kcsan_scoped_access *sa) { }
 
#endif /* CONFIG_KCSAN */
 
#ifdef __SANITIZE_THREAD__
/*
 * Only calls into the runtime when the particular compilation unit has KCSAN
 * instrumentation enabled. May be used in header files.
 */
#define kcsan_check_access __kcsan_check_access
 
/*
 * Only use these to disable KCSAN for accesses in the current compilation unit;
 * calls into libraries may still perform KCSAN checks.
 */
#define __kcsan_disable_current kcsan_disable_current
#define __kcsan_enable_current kcsan_enable_current_nowarn
#else /* __SANITIZE_THREAD__ */
static inline void kcsan_check_access(const volatile void *ptr, size_t size,
                      int type) { }
static inline void __kcsan_enable_current(void)  { }
static inline void __kcsan_disable_current(void) { }
#endif /* __SANITIZE_THREAD__ */
 
#if defined(CONFIG_KCSAN_WEAK_MEMORY) && defined(__SANITIZE_THREAD__)
/*
 * Normal barrier instrumentation is not done via explicit calls, but by mapping
 * to a repurposed __atomic_signal_fence(), which normally does not generate any
 * real instructions, but is still intercepted by fsanitize=thread. This means,
 * like any other compile-time instrumentation, barrier instrumentation can be
 * disabled with the __no_kcsan function attribute.
 *
 * Also see definition of __tsan_atomic_signal_fence() in kernel/kcsan/core.c.
 *
 * These are all macros, like <asm/barrier.h>, since some architectures use them
 * in non-static inline functions.
 */
#define __KCSAN_BARRIER_TO_SIGNAL_FENCE(name)                   \
    do {                                    \
        barrier();                          \
        __atomic_signal_fence(__KCSAN_BARRIER_TO_SIGNAL_FENCE_##name);  \
        barrier();                          \
    } while (0)
#define kcsan_mb()  __KCSAN_BARRIER_TO_SIGNAL_FENCE(mb)
#define kcsan_wmb() __KCSAN_BARRIER_TO_SIGNAL_FENCE(wmb)
#define kcsan_rmb() __KCSAN_BARRIER_TO_SIGNAL_FENCE(rmb)
#define kcsan_release() __KCSAN_BARRIER_TO_SIGNAL_FENCE(release)
#elif defined(CONFIG_KCSAN_WEAK_MEMORY) && defined(__KCSAN_INSTRUMENT_BARRIERS__)
#define kcsan_mb    __kcsan_mb
#define kcsan_wmb   __kcsan_wmb
#define kcsan_rmb   __kcsan_rmb
#define kcsan_release   __kcsan_release
#else /* CONFIG_KCSAN_WEAK_MEMORY && ... */
#define kcsan_mb()  do { } while (0)
#define kcsan_wmb() do { } while (0)
#define kcsan_rmb() do { } while (0)
#define kcsan_release() do { } while (0)
#endif /* CONFIG_KCSAN_WEAK_MEMORY && ... */
 
#define __kcsan_check_read(ptr, size) __kcsan_check_access(ptr, size, 0)
 
#define __kcsan_check_write(ptr, size)                                         \
    __kcsan_check_access(absolute_pointer(ptr), size, KCSAN_ACCESS_WRITE)
 
#define __kcsan_check_read_write(ptr, size)                                    \
    __kcsan_check_access(ptr, size, KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE)
 
#define kcsan_check_read(ptr, size) kcsan_check_access(ptr, size, 0)
 
#define kcsan_check_write(ptr, size)                                           \
    kcsan_check_access(absolute_pointer(ptr), size, KCSAN_ACCESS_WRITE)
 
#define kcsan_check_read_write(ptr, size)                                      \
    kcsan_check_access(ptr, size, KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE)
 
/*
 * Check for atomic accesses: if atomic accesses are not ignored, this simply
 * aliases to kcsan_check_access(), otherwise becomes a no-op.
 */
#ifdef CONFIG_KCSAN_IGNORE_ATOMICS
#define kcsan_check_atomic_read(...)        do { } while (0)
#define kcsan_check_atomic_write(...)       do { } while (0)
#define kcsan_check_atomic_read_write(...)  do { } while (0)
#else
#define kcsan_check_atomic_read(ptr, size)                                     \
    kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC)
#define kcsan_check_atomic_write(ptr, size)                                    \
    kcsan_check_access(absolute_pointer(ptr), size, KCSAN_ACCESS_ATOMIC | KCSAN_ACCESS_WRITE)
#define kcsan_check_atomic_read_write(ptr, size)                               \
    kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_COMPOUND)
#endif
 
#define ASSERT_EXCLUSIVE_WRITER(var)                                           \
    __kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_ASSERT)
 
/*
 * Helper macros for implementation of for ASSERT_EXCLUSIVE_*_SCOPED(). @id is
 * expected to be unique for the scope in which instances of kcsan_scoped_access
 * are declared.
 */
#define __kcsan_scoped_name(c, suffix) __kcsan_scoped_##c##suffix
#define __ASSERT_EXCLUSIVE_SCOPED(var, type, id)                               \
    struct kcsan_scoped_access __kcsan_scoped_name(id, _)                  \
        __kcsan_cleanup_scoped;                                        \
    struct kcsan_scoped_access *__kcsan_scoped_name(id, _dummy_p)          \
        __maybe_unused = kcsan_begin_scoped_access(                    \
            &(var), sizeof(var), KCSAN_ACCESS_SCOPED | (type),     \
            &__kcsan_scoped_name(id, _))
 
#define ASSERT_EXCLUSIVE_WRITER_SCOPED(var)                                    \
    __ASSERT_EXCLUSIVE_SCOPED(var, KCSAN_ACCESS_ASSERT, __COUNTER__)
 
#define ASSERT_EXCLUSIVE_ACCESS(var)                                           \
    __kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ASSERT)
 
#define ASSERT_EXCLUSIVE_ACCESS_SCOPED(var)                                    \
    __ASSERT_EXCLUSIVE_SCOPED(var, KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ASSERT, __COUNTER__)
 
#define ASSERT_EXCLUSIVE_BITS(var, mask)                                       \
    do {                                                                   \
        kcsan_set_access_mask(mask);                                   \
        __kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_ASSERT);\
        kcsan_set_access_mask(0);                                      \
        kcsan_atomic_next(1);                                          \
    } while (0)
 
#endif /* _LINUX_KCSAN_CHECKS_H */