drm_gpusvm.h

Go to the documentation of this file.

/* SPDX-License-Identifier: GPL-2.0-only OR MIT */
/*
 * Copyright © 2024 Intel Corporation
 */
 
#ifndef __DRM_GPUSVM_H__
#define __DRM_GPUSVM_H__
 
#include <linux/dma-mapping.h>
#include <linux/kref.h>
#include <linux/interval_tree.h>
#include <linux/mmu_notifier.h>
 
struct dev_pagemap_ops;
struct drm_device;
struct drm_gpusvm;
struct drm_gpusvm_notifier;
struct drm_gpusvm_ops;
struct drm_gpusvm_range;
struct drm_pagemap;
struct drm_pagemap_addr;
 
struct drm_gpusvm_ops {
    struct drm_gpusvm_notifier *(*notifier_alloc)(void);
 
    void (*notifier_free)(struct drm_gpusvm_notifier *notifier);
 
    struct drm_gpusvm_range *(*range_alloc)(struct drm_gpusvm *gpusvm);
 
    void (*range_free)(struct drm_gpusvm_range *range);
 
    void (*invalidate)(struct drm_gpusvm *gpusvm,
               struct drm_gpusvm_notifier *notifier,
               const struct mmu_notifier_range *mmu_range);
};
 
struct drm_gpusvm_notifier {
    struct drm_gpusvm *gpusvm;
    struct mmu_interval_notifier notifier;
    struct interval_tree_node itree;
    struct list_head entry;
    struct rb_root_cached root;
    struct list_head range_list;
    struct {
        u32 removed : 1;
    } flags;
};
 
struct drm_gpusvm_pages_flags {
    union {
        struct {
            /* All flags below must be set upon creation */
            u16 migrate_devmem : 1;
            /* All flags below must be set / cleared under notifier lock */
            u16 unmapped : 1;
            u16 partial_unmap : 1;
            u16 has_devmem_pages : 1;
            u16 has_dma_mapping : 1;
        };
        u16 __flags;
    };
};
 
struct drm_gpusvm_pages {
    struct drm_pagemap_addr *dma_addr;
    struct drm_pagemap *dpagemap;
    struct dma_iova_state state;
    unsigned long state_offset;
    unsigned long notifier_seq;
    struct drm_gpusvm_pages_flags flags;
};
 
struct drm_gpusvm_range {
    struct drm_gpusvm *gpusvm;
    struct drm_gpusvm_notifier *notifier;
    struct kref refcount;
    struct interval_tree_node itree;
    struct list_head entry;
    struct drm_gpusvm_pages pages;
};
 
struct drm_gpusvm {
    const char *name;
    struct drm_device *drm;
    struct mm_struct *mm;
    unsigned long mm_start;
    unsigned long mm_range;
    unsigned long notifier_size;
    const struct drm_gpusvm_ops *ops;
    const unsigned long *chunk_sizes;
    int num_chunks;
    struct rw_semaphore notifier_lock;
    struct rb_root_cached root;
    struct list_head notifier_list;
#ifdef CONFIG_LOCKDEP
    struct lockdep_map *lock_dep_map;
#endif
};
 
struct drm_gpusvm_ctx {
    void *device_private_page_owner;
    unsigned long check_pages_threshold;
    unsigned long timeslice_ms;
    unsigned int in_notifier :1;
    unsigned int read_only :1;
    unsigned int devmem_possible :1;
    unsigned int devmem_only :1;
    unsigned int allow_mixed :1;
};
 
int drm_gpusvm_init(struct drm_gpusvm *gpusvm,
            const char *name, struct drm_device *drm,
            struct mm_struct *mm,
            unsigned long mm_start, unsigned long mm_range,
            unsigned long notifier_size,
            const struct drm_gpusvm_ops *ops,
            const unsigned long *chunk_sizes, int num_chunks);
 
void drm_gpusvm_fini(struct drm_gpusvm *gpusvm);
 
void drm_gpusvm_free(struct drm_gpusvm *gpusvm);
 
unsigned long
drm_gpusvm_find_vma_start(struct drm_gpusvm *gpusvm,
              unsigned long start,
              unsigned long end);
 
struct drm_gpusvm_range *
drm_gpusvm_range_find_or_insert(struct drm_gpusvm *gpusvm,
                unsigned long fault_addr,
                unsigned long gpuva_start,
                unsigned long gpuva_end,
                const struct drm_gpusvm_ctx *ctx);
 
void drm_gpusvm_range_remove(struct drm_gpusvm *gpusvm,
                 struct drm_gpusvm_range *range);
 
int drm_gpusvm_range_evict(struct drm_gpusvm *gpusvm,
               struct drm_gpusvm_range *range);
 
struct drm_gpusvm_range *
drm_gpusvm_range_get(struct drm_gpusvm_range *range);
 
void drm_gpusvm_range_put(struct drm_gpusvm_range *range);
 
bool drm_gpusvm_range_pages_valid(struct drm_gpusvm *gpusvm,
                  struct drm_gpusvm_range *range);
 
int drm_gpusvm_range_get_pages(struct drm_gpusvm *gpusvm,
                   struct drm_gpusvm_range *range,
                   const struct drm_gpusvm_ctx *ctx);
 
void drm_gpusvm_range_unmap_pages(struct drm_gpusvm *gpusvm,
                  struct drm_gpusvm_range *range,
                  const struct drm_gpusvm_ctx *ctx);
 
bool drm_gpusvm_has_mapping(struct drm_gpusvm *gpusvm, unsigned long start,
                unsigned long end);
 
struct drm_gpusvm_notifier *
drm_gpusvm_notifier_find(struct drm_gpusvm *gpusvm, unsigned long start,
             unsigned long end);
 
struct drm_gpusvm_range *
drm_gpusvm_range_find(struct drm_gpusvm_notifier *notifier, unsigned long start,
              unsigned long end);
 
void drm_gpusvm_range_set_unmapped(struct drm_gpusvm_range *range,
                   const struct mmu_notifier_range *mmu_range);
 
int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
             struct drm_gpusvm_pages *svm_pages,
             struct mm_struct *mm,
             struct mmu_interval_notifier *notifier,
             unsigned long pages_start, unsigned long pages_end,
             const struct drm_gpusvm_ctx *ctx);
 
void drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm,
                struct drm_gpusvm_pages *svm_pages,
                unsigned long npages,
                const struct drm_gpusvm_ctx *ctx);
 
void drm_gpusvm_free_pages(struct drm_gpusvm *gpusvm,
               struct drm_gpusvm_pages *svm_pages,
               unsigned long npages);
 
enum drm_gpusvm_scan_result {
    DRM_GPUSVM_SCAN_UNPOPULATED,
    DRM_GPUSVM_SCAN_EQUAL,
    DRM_GPUSVM_SCAN_OTHER,
    DRM_GPUSVM_SCAN_SYSTEM,
    DRM_GPUSVM_SCAN_MIXED_DEVICE,
    DRM_GPUSVM_SCAN_MIXED,
};
 
enum drm_gpusvm_scan_result drm_gpusvm_scan_mm(struct drm_gpusvm_range *range,
                           void *dev_private_owner,
                           const struct dev_pagemap *pagemap);
 
#ifdef CONFIG_LOCKDEP
#define drm_gpusvm_driver_set_lock(gpusvm, lock) \
    do { \
        if (!WARN((gpusvm)->lock_dep_map, \
              "GPUSVM range lock should be set only once."))\
            (gpusvm)->lock_dep_map = &(lock)->dep_map;  \
    } while (0)
#else
#define drm_gpusvm_driver_set_lock(gpusvm, lock) do {} while (0)
#endif
 
#define drm_gpusvm_notifier_lock(gpusvm__)  \
    down_read(&(gpusvm__)->notifier_lock)
 
#define drm_gpusvm_notifier_unlock(gpusvm__)    \
    up_read(&(gpusvm__)->notifier_lock)
 
static inline unsigned long
drm_gpusvm_range_start(struct drm_gpusvm_range *range)
{
    return range->itree.start;
}
 
static inline unsigned long
drm_gpusvm_range_end(struct drm_gpusvm_range *range)
{
    return range->itree.last + 1;
}
 
static inline unsigned long
drm_gpusvm_range_size(struct drm_gpusvm_range *range)
{
    return drm_gpusvm_range_end(range) - drm_gpusvm_range_start(range);
}
 
static inline unsigned long
drm_gpusvm_notifier_start(struct drm_gpusvm_notifier *notifier)
{
    return notifier->itree.start;
}
 
static inline unsigned long
drm_gpusvm_notifier_end(struct drm_gpusvm_notifier *notifier)
{
    return notifier->itree.last + 1;
}
 
static inline unsigned long
drm_gpusvm_notifier_size(struct drm_gpusvm_notifier *notifier)
{
    return drm_gpusvm_notifier_end(notifier) -
        drm_gpusvm_notifier_start(notifier);
}
 
static inline struct drm_gpusvm_range *
__drm_gpusvm_range_next(struct drm_gpusvm_range *range)
{
    if (range && !list_is_last(&range->entry,
                   &range->notifier->range_list))
        return list_next_entry(range, entry);
 
    return NULL;
}
 
#define drm_gpusvm_for_each_range(range__, notifier__, start__, end__)  \
    for ((range__) = (range__) ?:                   \
         drm_gpusvm_range_find((notifier__), (start__), (end__));   \
         (range__) && (drm_gpusvm_range_start(range__) < (end__));  \
         (range__) = __drm_gpusvm_range_next(range__))
 
#define drm_gpusvm_for_each_range_safe(range__, next__, notifier__, start__, end__) \
    for ((range__) = drm_gpusvm_range_find((notifier__), (start__), (end__)),   \
         (next__) = __drm_gpusvm_range_next(range__);               \
         (range__) && (drm_gpusvm_range_start(range__) < (end__));          \
         (range__) = (next__), (next__) = __drm_gpusvm_range_next(range__))
 
static inline struct drm_gpusvm_notifier *
__drm_gpusvm_notifier_next(struct drm_gpusvm_notifier *notifier)
{
    if (notifier && !list_is_last(&notifier->entry,
                      &notifier->gpusvm->notifier_list))
        return list_next_entry(notifier, entry);
 
    return NULL;
}
 
#define drm_gpusvm_for_each_notifier(notifier__, gpusvm__, start__, end__)      \
    for ((notifier__) = drm_gpusvm_notifier_find((gpusvm__), (start__), (end__));   \
         (notifier__) && (drm_gpusvm_notifier_start(notifier__) < (end__));     \
         (notifier__) = __drm_gpusvm_notifier_next(notifier__))
 
#define drm_gpusvm_for_each_notifier_safe(notifier__, next__, gpusvm__, start__, end__) \
    for ((notifier__) = drm_gpusvm_notifier_find((gpusvm__), (start__), (end__)),   \
         (next__) = __drm_gpusvm_notifier_next(notifier__);             \
         (notifier__) && (drm_gpusvm_notifier_start(notifier__) < (end__));     \
         (notifier__) = (next__), (next__) = __drm_gpusvm_notifier_next(notifier__))
 
#endif /* __DRM_GPUSVM_H__ */