ww_mutex.h

Go to the documentation of this file.

/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Wound/Wait Mutexes: blocking mutual exclusion locks with deadlock avoidance
 *
 * Original mutex implementation started by Ingo Molnar:
 *
 *  Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
 *
 * Wait/Die implementation:
 *  Copyright (C) 2013 Canonical Ltd.
 * Choice of algorithm:
 *  Copyright (C) 2018 WMWare Inc.
 *
 * This file contains the main data structure and API definitions.
 */
 
#ifndef __LINUX_WW_MUTEX_H
#define __LINUX_WW_MUTEX_H
 
#include <linux/instruction_pointer.h>
#include <linux/mutex.h>
#include <linux/rtmutex.h>
 
#if defined(CONFIG_DEBUG_MUTEXES) || \
   (defined(CONFIG_PREEMPT_RT) && defined(CONFIG_DEBUG_RT_MUTEXES))
#define DEBUG_WW_MUTEXES
#endif
 
#ifndef CONFIG_PREEMPT_RT
#define WW_MUTEX_BASE           mutex
#define ww_mutex_base_init(l,n,k)   __mutex_init(l,n,k)
#define ww_mutex_base_is_locked(b)  mutex_is_locked((b))
#else
#define WW_MUTEX_BASE           rt_mutex
#define ww_mutex_base_init(l,n,k)   __rt_mutex_init(l,n,k)
#define ww_mutex_base_is_locked(b)  rt_mutex_base_is_locked(&(b)->rtmutex)
#endif
 
struct ww_class {
    atomic_long_t stamp;
    struct lock_class_key acquire_key;
    struct lock_class_key mutex_key;
    const char *acquire_name;
    const char *mutex_name;
    unsigned int is_wait_die;
};
 
context_lock_struct(ww_mutex) {
    struct WW_MUTEX_BASE base;
    struct ww_acquire_ctx *ctx;
#ifdef DEBUG_WW_MUTEXES
    struct ww_class *ww_class;
#endif
};
 
context_lock_struct(ww_acquire_ctx) {
    struct task_struct *task;
    unsigned long stamp;
    unsigned int acquired;
    unsigned short wounded;
    unsigned short is_wait_die;
#ifdef DEBUG_WW_MUTEXES
    unsigned int done_acquire;
    struct ww_class *ww_class;
    void *contending_lock;
#endif
#ifdef CONFIG_DEBUG_LOCK_ALLOC
    struct lockdep_map dep_map;
    struct lockdep_map first_lock_dep_map;
#endif
#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
    unsigned int deadlock_inject_interval;
    unsigned int deadlock_inject_countdown;
#endif
};
 
#define __WW_CLASS_INITIALIZER(ww_class, _is_wait_die)      \
        { .stamp = ATOMIC_LONG_INIT(0) \
        , .acquire_name = #ww_class "_acquire" \
        , .mutex_name = #ww_class "_mutex" \
        , .is_wait_die = _is_wait_die }
 
#define DEFINE_WD_CLASS(classname) \
    struct ww_class classname = __WW_CLASS_INITIALIZER(classname, 1)
 
#define DEFINE_WW_CLASS(classname) \
    struct ww_class classname = __WW_CLASS_INITIALIZER(classname, 0)
 
static inline void ww_mutex_init(struct ww_mutex *lock,
                 struct ww_class *ww_class)
{
    ww_mutex_base_init(&lock->base, ww_class->mutex_name, &ww_class->mutex_key);
    lock->ctx = NULL;
#ifdef DEBUG_WW_MUTEXES
    lock->ww_class = ww_class;
#endif
}
 
static inline void ww_acquire_init(struct ww_acquire_ctx *ctx,
                   struct ww_class *ww_class)
    __acquires(ctx) __no_context_analysis
{
    ctx->task = current;
    ctx->stamp = atomic_long_inc_return_relaxed(&ww_class->stamp);
    ctx->acquired = 0;
    ctx->wounded = false;
    ctx->is_wait_die = ww_class->is_wait_die;
#ifdef DEBUG_WW_MUTEXES
    ctx->ww_class = ww_class;
    ctx->done_acquire = 0;
    ctx->contending_lock = NULL;
#endif
#ifdef CONFIG_DEBUG_LOCK_ALLOC
    debug_check_no_locks_freed((void *)ctx, sizeof(*ctx));
    lockdep_init_map(&ctx->dep_map, ww_class->acquire_name,
             &ww_class->acquire_key, 0);
    lockdep_init_map_wait(&ctx->first_lock_dep_map, ww_class->mutex_name,
                  &ww_class->mutex_key, 0, LD_WAIT_SLEEP);
    mutex_acquire(&ctx->dep_map, 0, 0, _RET_IP_);
    mutex_acquire_nest(&ctx->first_lock_dep_map, 0, 0, &ctx->dep_map, _RET_IP_);
#endif
#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
    ctx->deadlock_inject_interval = 1;
    ctx->deadlock_inject_countdown = ctx->stamp & 0xf;
#endif
}
 
static inline void ww_acquire_done(struct ww_acquire_ctx *ctx)
    __must_hold(ctx)
{
#ifdef DEBUG_WW_MUTEXES
    lockdep_assert_held(ctx);
 
    DEBUG_LOCKS_WARN_ON(ctx->done_acquire);
    ctx->done_acquire = 1;
#endif
}
 
static inline void ww_acquire_fini(struct ww_acquire_ctx *ctx)
    __releases(ctx) __no_context_analysis
{
#ifdef CONFIG_DEBUG_LOCK_ALLOC
    mutex_release(&ctx->first_lock_dep_map, _THIS_IP_);
    mutex_release(&ctx->dep_map, _THIS_IP_);
#endif
#ifdef DEBUG_WW_MUTEXES
    DEBUG_LOCKS_WARN_ON(ctx->acquired);
    if (!IS_ENABLED(CONFIG_PROVE_LOCKING))
        /*
         * lockdep will normally handle this,
         * but fail without anyway
         */
        ctx->done_acquire = 1;
 
    if (!IS_ENABLED(CONFIG_DEBUG_LOCK_ALLOC))
        /* ensure ww_acquire_fini will still fail if called twice */
        ctx->acquired = ~0U;
#endif
}
 
extern int /* __must_check */ ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
    __cond_acquires(0, lock) __must_hold(ctx);
 
extern int __must_check ww_mutex_lock_interruptible(struct ww_mutex *lock,
                            struct ww_acquire_ctx *ctx)
    __cond_acquires(0, lock) __must_hold(ctx);
 
static inline void
ww_mutex_lock_slow(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
    __acquires(lock) __must_hold(ctx) __no_context_analysis
{
    int ret;
#ifdef DEBUG_WW_MUTEXES
    DEBUG_LOCKS_WARN_ON(!ctx->contending_lock);
#endif
    ret = ww_mutex_lock(lock, ctx);
    (void)ret;
}
 
static inline int __must_check
ww_mutex_lock_slow_interruptible(struct ww_mutex *lock,
                 struct ww_acquire_ctx *ctx)
    __cond_acquires(0, lock) __must_hold(ctx)
{
#ifdef DEBUG_WW_MUTEXES
    DEBUG_LOCKS_WARN_ON(!ctx->contending_lock);
#endif
    return ww_mutex_lock_interruptible(lock, ctx);
}
 
extern void ww_mutex_unlock(struct ww_mutex *lock) __releases(lock);
 
extern int __must_check ww_mutex_trylock(struct ww_mutex *lock,
                     struct ww_acquire_ctx *ctx)
    __cond_acquires(true, lock) __must_hold(ctx);
 
/***
 * ww_mutex_destroy - mark a w/w mutex unusable
 * @lock: the mutex to be destroyed
 *
 * This function marks the mutex uninitialized, and any subsequent
 * use of the mutex is forbidden. The mutex must not be locked when
 * this function is called.
 */
static inline void ww_mutex_destroy(struct ww_mutex *lock)
    __must_not_hold(lock)
{
#ifndef CONFIG_PREEMPT_RT
    mutex_destroy(&lock->base);
#endif
}
 
static inline bool ww_mutex_is_locked(struct ww_mutex *lock)
{
    return ww_mutex_base_is_locked(&lock->base);
}
 
#endif