gpu_scheduler.h

Go to the documentation of this file.

/*
 * Copyright 2015 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 */
 
#ifndef _DRM_GPU_SCHEDULER_H_
#define _DRM_GPU_SCHEDULER_H_
 
#include <drm/spsc_queue.h>
#include <linux/average.h>
#include <linux/dma-fence.h>
#include <linux/completion.h>
#include <linux/xarray.h>
#include <linux/workqueue.h>
 
DECLARE_EWMA(drm_sched_avgtime, 6, 4);
 
#define MAX_WAIT_SCHED_ENTITY_Q_EMPTY msecs_to_jiffies(1000)
 
#define DRM_SCHED_FENCE_DONT_PIPELINE   DMA_FENCE_FLAG_USER_BITS
 
#define DRM_SCHED_FENCE_FLAG_HAS_DEADLINE_BIT   (DMA_FENCE_FLAG_USER_BITS + 1)
 
enum dma_resv_usage;
struct dma_resv;
struct drm_gem_object;
 
struct drm_gpu_scheduler;
struct drm_sched_rq;
 
struct drm_file;
 
/* These are often used as an (initial) index
 * to an array, and as such should start at 0.
 */
enum drm_sched_priority {
    DRM_SCHED_PRIORITY_INVALID = -1, /* Internal marker - do not use. */
    DRM_SCHED_PRIORITY_KERNEL,
    DRM_SCHED_PRIORITY_HIGH,
    DRM_SCHED_PRIORITY_NORMAL,
    DRM_SCHED_PRIORITY_LOW,
 
    DRM_SCHED_PRIORITY_COUNT
};
 
struct drm_sched_entity_stats;
 
struct drm_sched_entity {
    struct list_head        list;
 
    spinlock_t          lock;
 
    struct drm_sched_rq     *rq;
 
    struct drm_sched_entity_stats   *stats;
 
    struct drm_gpu_scheduler        **sched_list;
 
    unsigned int                    num_sched_list;
 
    enum drm_sched_priority         priority;
 
    enum drm_sched_priority         rq_priority;
 
    ktime_t             rr_ts;
 
    struct spsc_queue       job_queue;
 
    atomic_t            fence_seq;
 
    uint64_t            fence_context;
 
    struct dma_fence        *dependency;
 
    struct dma_fence_cb     cb;
 
    atomic_t            *guilty;
 
    struct dma_fence __rcu      *last_scheduled;
 
    struct task_struct      *last_user;
 
    bool                stopped;
 
    struct completion       entity_idle;
 
    ktime_t             oldest_job_waiting;
 
    struct rb_node          rb_tree_node;
 
};
 
struct drm_sched_rq {
    struct drm_gpu_scheduler    *sched;
 
    spinlock_t          lock;
    /* Following members are protected by the @lock: */
    ktime_t             rr_ts;
    struct list_head        entities;
    struct rb_root_cached       rb_tree_root;
    enum drm_sched_priority     head_prio;
};
 
struct drm_sched_fence {
    struct dma_fence        scheduled;
 
    struct dma_fence        finished;
 
    ktime_t             deadline;
 
    struct dma_fence        *parent;
    struct drm_gpu_scheduler    *sched;
    spinlock_t          lock;
    void                *owner;
 
    uint64_t            drm_client_id;
};
 
struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f);
 
struct drm_sched_job {
    ktime_t                         submit_ts;
 
    struct drm_gpu_scheduler    *sched;
 
    struct drm_sched_fence      *s_fence;
    struct drm_sched_entity         *entity;
 
    struct drm_sched_entity_stats   *entity_stats;
 
    enum drm_sched_priority     s_priority;
    u32             credits;
    unsigned int            last_dependency;
    atomic_t            karma;
 
    struct spsc_node        queue_node;
    struct list_head        list;
 
    /*
     * work is used only after finish_cb has been used and will not be
     * accessed anymore.
     */
    union {
        struct dma_fence_cb finish_cb;
        struct work_struct  work;
    };
 
    struct dma_fence_cb     cb;
 
    struct xarray           dependencies;
};
 
enum drm_gpu_sched_stat {
    DRM_GPU_SCHED_STAT_NONE,
    DRM_GPU_SCHED_STAT_RESET,
    DRM_GPU_SCHED_STAT_ENODEV,
    DRM_GPU_SCHED_STAT_NO_HANG,
};
 
struct drm_sched_backend_ops {
    struct dma_fence *(*prepare_job)(struct drm_sched_job *sched_job,
                     struct drm_sched_entity *s_entity);
 
    struct dma_fence *(*run_job)(struct drm_sched_job *sched_job);
 
    enum drm_gpu_sched_stat (*timedout_job)(struct drm_sched_job *sched_job);
 
    void (*free_job)(struct drm_sched_job *sched_job);
 
    void (*cancel_job)(struct drm_sched_job *sched_job);
};
 
struct drm_gpu_scheduler {
    const struct drm_sched_backend_ops  *ops;
    u32             credit_limit;
    atomic_t            credit_count;
    long                timeout;
    const char          *name;
    u32                             num_rqs;
    u32                             num_user_rqs;
    struct drm_sched_rq             **sched_rq;
    wait_queue_head_t       job_scheduled;
    atomic64_t          job_id_count;
    struct workqueue_struct     *submit_wq;
    struct workqueue_struct     *timeout_wq;
    struct ewma_drm_sched_avgtime   avg_job_us;
    struct work_struct      work_run_job;
    struct work_struct      work_free_job;
    struct delayed_work     work_tdr;
    struct list_head        pending_list;
    spinlock_t          job_list_lock;
    int             hang_limit;
    atomic_t                        *score;
    atomic_t                        _score;
    bool                ready;
    bool                free_guilty;
    bool                pause_submit;
    bool                own_submit_wq;
    struct device           *dev;
};
 
struct drm_sched_init_args {
    const struct drm_sched_backend_ops *ops;
    struct workqueue_struct *submit_wq;
    struct workqueue_struct *timeout_wq;
    u32 num_rqs;
    u32 credit_limit;
    unsigned int hang_limit;
    long timeout;
    atomic_t *score;
    const char *name;
    struct device *dev;
};
 
/* Scheduler operations */
 
int drm_sched_init(struct drm_gpu_scheduler *sched,
           const struct drm_sched_init_args *args);
 
void drm_sched_fini(struct drm_gpu_scheduler *sched);
 
unsigned long drm_sched_suspend_timeout(struct drm_gpu_scheduler *sched);
void drm_sched_resume_timeout(struct drm_gpu_scheduler *sched,
                  unsigned long remaining);
void drm_sched_tdr_queue_imm(struct drm_gpu_scheduler *sched);
bool drm_sched_wqueue_ready(struct drm_gpu_scheduler *sched);
void drm_sched_wqueue_stop(struct drm_gpu_scheduler *sched);
void drm_sched_wqueue_start(struct drm_gpu_scheduler *sched);
void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad);
void drm_sched_start(struct drm_gpu_scheduler *sched, int errno);
void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched);
void drm_sched_fault(struct drm_gpu_scheduler *sched);
bool drm_sched_is_stopped(struct drm_gpu_scheduler *sched);
 
struct drm_gpu_scheduler *
drm_sched_pick_best(struct drm_gpu_scheduler **sched_list,
            unsigned int num_sched_list);
 
/* Jobs */
 
int drm_sched_job_init(struct drm_sched_job *job,
               struct drm_sched_entity *entity,
               u32 credits, void *owner,
               u64 drm_client_id);
void drm_sched_job_arm(struct drm_sched_job *job);
void drm_sched_entity_push_job(struct drm_sched_job *sched_job);
int drm_sched_job_add_dependency(struct drm_sched_job *job,
                 struct dma_fence *fence);
int drm_sched_job_add_syncobj_dependency(struct drm_sched_job *job,
                     struct drm_file *file,
                     u32 handle,
                     u32 point);
int drm_sched_job_add_resv_dependencies(struct drm_sched_job *job,
                    struct dma_resv *resv,
                    enum dma_resv_usage usage);
int drm_sched_job_add_implicit_dependencies(struct drm_sched_job *job,
                        struct drm_gem_object *obj,
                        bool write);
bool drm_sched_job_has_dependency(struct drm_sched_job *job,
                  struct dma_fence *fence);
void drm_sched_job_cleanup(struct drm_sched_job *job);
void drm_sched_increase_karma(struct drm_sched_job *bad);
bool drm_sched_job_is_signaled(struct drm_sched_job *job);
 
static inline bool drm_sched_invalidate_job(struct drm_sched_job *s_job,
                        int threshold)
{
    return s_job && atomic_inc_return(&s_job->karma) > threshold;
}
 
/* Entities */
 
int drm_sched_entity_init(struct drm_sched_entity *entity,
              enum drm_sched_priority priority,
              struct drm_gpu_scheduler **sched_list,
              unsigned int num_sched_list,
              atomic_t *guilty);
long drm_sched_entity_flush(struct drm_sched_entity *entity, long timeout);
void drm_sched_entity_kill(struct drm_sched_entity *entity);
void drm_sched_entity_fini(struct drm_sched_entity *entity);
void drm_sched_entity_destroy(struct drm_sched_entity *entity);
void drm_sched_entity_set_priority(struct drm_sched_entity *entity,
                   enum drm_sched_priority priority);
int drm_sched_entity_error(struct drm_sched_entity *entity);
void drm_sched_entity_modify_sched(struct drm_sched_entity *entity,
                   struct drm_gpu_scheduler **sched_list,
                   unsigned int num_sched_list);
 
struct drm_sched_pending_job_iter {
    struct drm_gpu_scheduler *sched;
};
 
/* Drivers should never call this directly */
static inline struct drm_sched_pending_job_iter
__drm_sched_pending_job_iter_begin(struct drm_gpu_scheduler *sched)
{
    struct drm_sched_pending_job_iter iter = {
        .sched = sched,
    };
 
    WARN_ON(!drm_sched_is_stopped(sched));
    return iter;
}
 
/* Drivers should never call this directly */
static inline void
__drm_sched_pending_job_iter_end(const struct drm_sched_pending_job_iter iter)
{
    WARN_ON(!drm_sched_is_stopped(iter.sched));
}
 
DEFINE_CLASS(drm_sched_pending_job_iter, struct drm_sched_pending_job_iter,
         __drm_sched_pending_job_iter_end(_T),
         __drm_sched_pending_job_iter_begin(__sched),
         struct drm_gpu_scheduler *__sched);
static inline void *
class_drm_sched_pending_job_iter_lock_ptr(class_drm_sched_pending_job_iter_t *_T)
{ return _T; }
#define class_drm_sched_pending_job_iter_is_conditional false
 
#define drm_sched_for_each_pending_job(__job, __sched, __entity)        \
    scoped_guard(drm_sched_pending_job_iter, (__sched))         \
        list_for_each_entry((__job), &(__sched)->pending_list, list)    \
            for_each_if(!(__entity) || (__job)->entity == (__entity))
 
#endif