From 33bf0d7437eaab1027c13bc117c5c186fd0d7cf9 Mon Sep 17 00:00:00 2001 From: Jordan Justen Date: Fri, 22 Jul 2022 10:30:14 -0700 Subject: [PATCH 1/5] drm-uapi/i915_drm.h: Update from drm-next (2022-07-22) git://anongit.freedesktop.org/drm/drm 417c1c1963549e9a48b83ada59d90258e38c6594 Signed-off-by: Jordan Justen Part-of: --- include/drm-uapi/i915_drm.h | 387 ++++++++++++++++++++++++++++-------- 1 file changed, 300 insertions(+), 87 deletions(-) diff --git a/include/drm-uapi/i915_drm.h b/include/drm-uapi/i915_drm.h index b4efc96c2edc..25834de67047 100644 --- a/include/drm-uapi/i915_drm.h +++ b/include/drm-uapi/i915_drm.h @@ -751,14 +751,27 @@ typedef struct drm_i915_irq_wait { /* Must be kept compact -- no holes and well documented */ -typedef struct drm_i915_getparam { +/** + * struct drm_i915_getparam - Driver parameter query structure. + */ +struct drm_i915_getparam { + /** @param: Driver parameter to query. */ __s32 param; - /* + + /** + * @value: Address of memory where queried value should be put. + * * WARNING: Using pointers instead of fixed-size u64 means we need to write * compat32 code. Don't repeat this mistake. */ int *value; -} drm_i915_getparam_t; +}; + +/** + * typedef drm_i915_getparam_t - Driver parameter query structure. + * See struct drm_i915_getparam. + */ +typedef struct drm_i915_getparam drm_i915_getparam_t; /* Ioctl to set kernel params: */ @@ -1239,76 +1252,119 @@ struct drm_i915_gem_exec_object2 { __u64 rsvd2; }; +/** + * struct drm_i915_gem_exec_fence - An input or output fence for the execbuf + * ioctl. + * + * The request will wait for input fence to signal before submission. + * + * The returned output fence will be signaled after the completion of the + * request. + */ struct drm_i915_gem_exec_fence { - /** - * User's handle for a drm_syncobj to wait on or signal. - */ + /** @handle: User's handle for a drm_syncobj to wait on or signal. */ __u32 handle; + /** + * @flags: Supported flags are: + * + * I915_EXEC_FENCE_WAIT: + * Wait for the input fence before request submission. + * + * I915_EXEC_FENCE_SIGNAL: + * Return request completion fence as output + */ + __u32 flags; #define I915_EXEC_FENCE_WAIT (1<<0) #define I915_EXEC_FENCE_SIGNAL (1<<1) #define __I915_EXEC_FENCE_UNKNOWN_FLAGS (-(I915_EXEC_FENCE_SIGNAL << 1)) - __u32 flags; }; -/* - * See drm_i915_gem_execbuffer_ext_timeline_fences. - */ -#define DRM_I915_GEM_EXECBUFFER_EXT_TIMELINE_FENCES 0 - -/* +/** + * struct drm_i915_gem_execbuffer_ext_timeline_fences - Timeline fences + * for execbuf ioctl. + * * This structure describes an array of drm_syncobj and associated points for * timeline variants of drm_syncobj. It is invalid to append this structure to * the execbuf if I915_EXEC_FENCE_ARRAY is set. */ struct drm_i915_gem_execbuffer_ext_timeline_fences { +#define DRM_I915_GEM_EXECBUFFER_EXT_TIMELINE_FENCES 0 + /** @base: Extension link. See struct i915_user_extension. */ struct i915_user_extension base; /** - * Number of element in the handles_ptr & value_ptr arrays. + * @fence_count: Number of elements in the @handles_ptr & @value_ptr + * arrays. */ __u64 fence_count; /** - * Pointer to an array of struct drm_i915_gem_exec_fence of length - * fence_count. + * @handles_ptr: Pointer to an array of struct drm_i915_gem_exec_fence + * of length @fence_count. */ __u64 handles_ptr; /** - * Pointer to an array of u64 values of length fence_count. Values - * must be 0 for a binary drm_syncobj. A Value of 0 for a timeline - * drm_syncobj is invalid as it turns a drm_syncobj into a binary one. + * @values_ptr: Pointer to an array of u64 values of length + * @fence_count. + * Values must be 0 for a binary drm_syncobj. A Value of 0 for a + * timeline drm_syncobj is invalid as it turns a drm_syncobj into a + * binary one. */ __u64 values_ptr; }; +/** + * struct drm_i915_gem_execbuffer2 - Structure for DRM_I915_GEM_EXECBUFFER2 + * ioctl. + */ struct drm_i915_gem_execbuffer2 { - /** - * List of gem_exec_object2 structs - */ + /** @buffers_ptr: Pointer to a list of gem_exec_object2 structs */ __u64 buffers_ptr; + + /** @buffer_count: Number of elements in @buffers_ptr array */ __u32 buffer_count; - /** Offset in the batchbuffer to start execution from. */ + /** + * @batch_start_offset: Offset in the batchbuffer to start execution + * from. + */ __u32 batch_start_offset; - /** Bytes used in batchbuffer from batch_start_offset */ + + /** + * @batch_len: Length in bytes of the batch buffer, starting from the + * @batch_start_offset. If 0, length is assumed to be the batch buffer + * object size. + */ __u32 batch_len; + + /** @DR1: deprecated */ __u32 DR1; + + /** @DR4: deprecated */ __u32 DR4; + + /** @num_cliprects: See @cliprects_ptr */ __u32 num_cliprects; + /** - * This is a struct drm_clip_rect *cliprects if I915_EXEC_FENCE_ARRAY - * & I915_EXEC_USE_EXTENSIONS are not set. + * @cliprects_ptr: Kernel clipping was a DRI1 misfeature. + * + * It is invalid to use this field if I915_EXEC_FENCE_ARRAY or + * I915_EXEC_USE_EXTENSIONS flags are not set. * * If I915_EXEC_FENCE_ARRAY is set, then this is a pointer to an array - * of struct drm_i915_gem_exec_fence and num_cliprects is the length - * of the array. + * of &drm_i915_gem_exec_fence and @num_cliprects is the length of the + * array. * * If I915_EXEC_USE_EXTENSIONS is set, then this is a pointer to a - * single struct i915_user_extension and num_cliprects is 0. + * single &i915_user_extension and num_cliprects is 0. */ __u64 cliprects_ptr; + + /** @flags: Execbuf flags */ + __u64 flags; #define I915_EXEC_RING_MASK (0x3f) #define I915_EXEC_DEFAULT (0<<0) #define I915_EXEC_RENDER (1<<0) @@ -1326,10 +1382,6 @@ struct drm_i915_gem_execbuffer2 { #define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */ #define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6) #define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */ - __u64 flags; - __u64 rsvd1; /* now used for context info */ - __u64 rsvd2; -}; /** Resets the SO write offset registers for transform feedback on gen7. */ #define I915_EXEC_GEN7_SOL_RESET (1<<8) @@ -1432,9 +1484,23 @@ struct drm_i915_gem_execbuffer2 { * drm_i915_gem_execbuffer_ext enum. */ #define I915_EXEC_USE_EXTENSIONS (1 << 21) - #define __I915_EXEC_UNKNOWN_FLAGS (-(I915_EXEC_USE_EXTENSIONS << 1)) + /** @rsvd1: Context id */ + __u64 rsvd1; + + /** + * @rsvd2: in and out sync_file file descriptors. + * + * When I915_EXEC_FENCE_IN or I915_EXEC_FENCE_SUBMIT flag is set, the + * lower 32 bits of this field will have the in sync_file fd (input). + * + * When I915_EXEC_FENCE_OUT flag is set, the upper 32 bits of this + * field will have the out sync_file fd (output). + */ + __u64 rsvd2; +}; + #define I915_EXEC_CONTEXT_ID_MASK (0xffffffff) #define i915_execbuffer2_set_context_id(eb2, context) \ (eb2).rsvd1 = context & I915_EXEC_CONTEXT_ID_MASK @@ -1814,19 +1880,58 @@ struct drm_i915_gem_context_create { __u32 pad; }; +/** + * struct drm_i915_gem_context_create_ext - Structure for creating contexts. + */ struct drm_i915_gem_context_create_ext { - __u32 ctx_id; /* output: id of new context*/ + /** @ctx_id: Id of the created context (output) */ + __u32 ctx_id; + + /** + * @flags: Supported flags are: + * + * I915_CONTEXT_CREATE_FLAGS_USE_EXTENSIONS: + * + * Extensions may be appended to this structure and driver must check + * for those. See @extensions. + * + * I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE + * + * Created context will have single timeline. + */ __u32 flags; #define I915_CONTEXT_CREATE_FLAGS_USE_EXTENSIONS (1u << 0) #define I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE (1u << 1) #define I915_CONTEXT_CREATE_FLAGS_UNKNOWN \ (-(I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE << 1)) + + /** + * @extensions: Zero-terminated chain of extensions. + * + * I915_CONTEXT_CREATE_EXT_SETPARAM: + * Context parameter to set or query during context creation. + * See struct drm_i915_gem_context_create_ext_setparam. + * + * I915_CONTEXT_CREATE_EXT_CLONE: + * This extension has been removed. On the off chance someone somewhere + * has attempted to use it, never re-use this extension number. + */ __u64 extensions; +#define I915_CONTEXT_CREATE_EXT_SETPARAM 0 +#define I915_CONTEXT_CREATE_EXT_CLONE 1 }; +/** + * struct drm_i915_gem_context_param - Context parameter to set or query. + */ struct drm_i915_gem_context_param { + /** @ctx_id: Context id */ __u32 ctx_id; + + /** @size: Size of the parameter @value */ __u32 size; + + /** @param: Parameter to set or query */ __u64 param; #define I915_CONTEXT_PARAM_BAN_PERIOD 0x1 /* I915_CONTEXT_PARAM_NO_ZEROMAP has been removed. On the off chance @@ -1973,6 +2078,7 @@ struct drm_i915_gem_context_param { #define I915_CONTEXT_PARAM_PROTECTED_CONTENT 0xd /* Must be kept compact -- no holes and well documented */ + /** @value: Context parameter value to be set or queried */ __u64 value; }; @@ -2371,23 +2477,29 @@ struct i915_context_param_engines { struct i915_engine_class_instance engines[N__]; \ } __attribute__((packed)) name__ +/** + * struct drm_i915_gem_context_create_ext_setparam - Context parameter + * to set or query during context creation. + */ struct drm_i915_gem_context_create_ext_setparam { -#define I915_CONTEXT_CREATE_EXT_SETPARAM 0 + /** @base: Extension link. See struct i915_user_extension. */ struct i915_user_extension base; + + /** + * @param: Context parameter to set or query. + * See struct drm_i915_gem_context_param. + */ struct drm_i915_gem_context_param param; }; -/* This API has been removed. On the off chance someone somewhere has - * attempted to use it, never re-use this extension number. - */ -#define I915_CONTEXT_CREATE_EXT_CLONE 1 - struct drm_i915_gem_context_destroy { __u32 ctx_id; __u32 pad; }; -/* +/** + * struct drm_i915_gem_vm_control - Structure to create or destroy VM. + * * DRM_I915_GEM_VM_CREATE - * * Create a new virtual memory address space (ppGTT) for use within a context @@ -2397,20 +2509,23 @@ struct drm_i915_gem_context_destroy { * The id of new VM (bound to the fd) for use with I915_CONTEXT_PARAM_VM is * returned in the outparam @id. * - * No flags are defined, with all bits reserved and must be zero. - * * An extension chain maybe provided, starting with @extensions, and terminated * by the @next_extension being 0. Currently, no extensions are defined. * * DRM_I915_GEM_VM_DESTROY - * - * Destroys a previously created VM id, specified in @id. + * Destroys a previously created VM id, specified in @vm_id. * * No extensions or flags are allowed currently, and so must be zero. */ struct drm_i915_gem_vm_control { + /** @extensions: Zero-terminated chain of extensions. */ __u64 extensions; + + /** @flags: reserved for future usage, currently MBZ */ __u32 flags; + + /** @vm_id: Id of the VM created or to be destroyed */ __u32 vm_id; }; @@ -3207,36 +3322,6 @@ struct drm_i915_gem_memory_class_instance { * struct drm_i915_memory_region_info - Describes one region as known to the * driver. * - * Note that we reserve some stuff here for potential future work. As an example - * we might want expose the capabilities for a given region, which could include - * things like if the region is CPU mappable/accessible, what are the supported - * mapping types etc. - * - * Note that to extend struct drm_i915_memory_region_info and struct - * drm_i915_query_memory_regions in the future the plan is to do the following: - * - * .. code-block:: C - * - * struct drm_i915_memory_region_info { - * struct drm_i915_gem_memory_class_instance region; - * union { - * __u32 rsvd0; - * __u32 new_thing1; - * }; - * ... - * union { - * __u64 rsvd1[8]; - * struct { - * __u64 new_thing2; - * __u64 new_thing3; - * ... - * }; - * }; - * }; - * - * With this things should remain source compatible between versions for - * userspace, even as we add new fields. - * * Note this is using both struct drm_i915_query_item and struct drm_i915_query. * For this new query we are adding the new query id DRM_I915_QUERY_MEMORY_REGIONS * at &drm_i915_query_item.query_id. @@ -3248,14 +3333,81 @@ struct drm_i915_memory_region_info { /** @rsvd0: MBZ */ __u32 rsvd0; - /** @probed_size: Memory probed by the driver (-1 = unknown) */ + /** + * @probed_size: Memory probed by the driver + * + * Note that it should not be possible to ever encounter a zero value + * here, also note that no current region type will ever return -1 here. + * Although for future region types, this might be a possibility. The + * same applies to the other size fields. + */ __u64 probed_size; - /** @unallocated_size: Estimate of memory remaining (-1 = unknown) */ + /** + * @unallocated_size: Estimate of memory remaining + * + * Requires CAP_PERFMON or CAP_SYS_ADMIN to get reliable accounting. + * Without this (or if this is an older kernel) the value here will + * always equal the @probed_size. Note this is only currently tracked + * for I915_MEMORY_CLASS_DEVICE regions (for other types the value here + * will always equal the @probed_size). + */ __u64 unallocated_size; - /** @rsvd1: MBZ */ - __u64 rsvd1[8]; + union { + /** @rsvd1: MBZ */ + __u64 rsvd1[8]; + struct { + /** + * @probed_cpu_visible_size: Memory probed by the driver + * that is CPU accessible. + * + * This will be always be <= @probed_size, and the + * remainder (if there is any) will not be CPU + * accessible. + * + * On systems without small BAR, the @probed_size will + * always equal the @probed_cpu_visible_size, since all + * of it will be CPU accessible. + * + * Note this is only tracked for + * I915_MEMORY_CLASS_DEVICE regions (for other types the + * value here will always equal the @probed_size). + * + * Note that if the value returned here is zero, then + * this must be an old kernel which lacks the relevant + * small-bar uAPI support (including + * I915_GEM_CREATE_EXT_FLAG_NEEDS_CPU_ACCESS), but on + * such systems we should never actually end up with a + * small BAR configuration, assuming we are able to load + * the kernel module. Hence it should be safe to treat + * this the same as when @probed_cpu_visible_size == + * @probed_size. + */ + __u64 probed_cpu_visible_size; + + /** + * @unallocated_cpu_visible_size: Estimate of CPU + * visible memory remaining. + * + * Note this is only tracked for + * I915_MEMORY_CLASS_DEVICE regions (for other types the + * value here will always equal the + * @probed_cpu_visible_size). + * + * Requires CAP_PERFMON or CAP_SYS_ADMIN to get reliable + * accounting. Without this the value here will always + * equal the @probed_cpu_visible_size. Note this is only + * currently tracked for I915_MEMORY_CLASS_DEVICE + * regions (for other types the value here will also + * always equal the @probed_cpu_visible_size). + * + * If this is an older kernel the value here will be + * zero, see also @probed_cpu_visible_size. + */ + __u64 unallocated_cpu_visible_size; + }; + }; }; /** @@ -3329,11 +3481,11 @@ struct drm_i915_query_memory_regions { * struct drm_i915_gem_create_ext - Existing gem_create behaviour, with added * extension support using struct i915_user_extension. * - * Note that in the future we want to have our buffer flags here, at least for - * the stuff that is immutable. Previously we would have two ioctls, one to - * create the object with gem_create, and another to apply various parameters, - * however this creates some ambiguity for the params which are considered - * immutable. Also in general we're phasing out the various SET/GET ioctls. + * Note that new buffer flags should be added here, at least for the stuff that + * is immutable. Previously we would have two ioctls, one to create the object + * with gem_create, and another to apply various parameters, however this + * creates some ambiguity for the params which are considered immutable. Also in + * general we're phasing out the various SET/GET ioctls. */ struct drm_i915_gem_create_ext { /** @@ -3341,7 +3493,6 @@ struct drm_i915_gem_create_ext { * * The (page-aligned) allocated size for the object will be returned. * - * * DG2 64K min page size implications: * * On discrete platforms, starting from DG2, we have to contend with GTT @@ -3353,7 +3504,9 @@ struct drm_i915_gem_create_ext { * * Note that the returned size here will always reflect any required * rounding up done by the kernel, i.e 4K will now become 64K on devices - * such as DG2. + * such as DG2. The kernel will always select the largest minimum + * page-size for the set of possible placements as the value to use when + * rounding up the @size. * * Special DG2 GTT address alignment requirement: * @@ -3377,14 +3530,58 @@ struct drm_i915_gem_create_ext { * is deemed to be a good compromise. */ __u64 size; + /** * @handle: Returned handle for the object. * * Object handles are nonzero. */ __u32 handle; - /** @flags: MBZ */ + + /** + * @flags: Optional flags. + * + * Supported values: + * + * I915_GEM_CREATE_EXT_FLAG_NEEDS_CPU_ACCESS - Signal to the kernel that + * the object will need to be accessed via the CPU. + * + * Only valid when placing objects in I915_MEMORY_CLASS_DEVICE, and only + * strictly required on configurations where some subset of the device + * memory is directly visible/mappable through the CPU (which we also + * call small BAR), like on some DG2+ systems. Note that this is quite + * undesirable, but due to various factors like the client CPU, BIOS etc + * it's something we can expect to see in the wild. See + * &drm_i915_memory_region_info.probed_cpu_visible_size for how to + * determine if this system applies. + * + * Note that one of the placements MUST be I915_MEMORY_CLASS_SYSTEM, to + * ensure the kernel can always spill the allocation to system memory, + * if the object can't be allocated in the mappable part of + * I915_MEMORY_CLASS_DEVICE. + * + * Also note that since the kernel only supports flat-CCS on objects + * that can *only* be placed in I915_MEMORY_CLASS_DEVICE, we therefore + * don't support I915_GEM_CREATE_EXT_FLAG_NEEDS_CPU_ACCESS together with + * flat-CCS. + * + * Without this hint, the kernel will assume that non-mappable + * I915_MEMORY_CLASS_DEVICE is preferred for this object. Note that the + * kernel can still migrate the object to the mappable part, as a last + * resort, if userspace ever CPU faults this object, but this might be + * expensive, and so ideally should be avoided. + * + * On older kernels which lack the relevant small-bar uAPI support (see + * also &drm_i915_memory_region_info.probed_cpu_visible_size), + * usage of the flag will result in an error, but it should NEVER be + * possible to end up with a small BAR configuration, assuming we can + * also successfully load the i915 kernel module. In such cases the + * entire I915_MEMORY_CLASS_DEVICE region will be CPU accessible, and as + * such there are zero restrictions on where the object can be placed. + */ +#define I915_GEM_CREATE_EXT_FLAG_NEEDS_CPU_ACCESS (1 << 0) __u32 flags; + /** * @extensions: The chain of extensions to apply to this object. * @@ -3443,6 +3640,22 @@ struct drm_i915_gem_create_ext { * At which point we get the object handle in &drm_i915_gem_create_ext.handle, * along with the final object size in &drm_i915_gem_create_ext.size, which * should account for any rounding up, if required. + * + * Note that userspace has no means of knowing the current backing region + * for objects where @num_regions is larger than one. The kernel will only + * ensure that the priority order of the @regions array is honoured, either + * when initially placing the object, or when moving memory around due to + * memory pressure + * + * On Flat-CCS capable HW, compression is supported for the objects residing + * in I915_MEMORY_CLASS_DEVICE. When such objects (compressed) have other + * memory class in @regions and migrated (by i915, due to memory + * constraints) to the non I915_MEMORY_CLASS_DEVICE region, then i915 needs to + * decompress the content. But i915 doesn't have the required information to + * decompress the userspace compressed objects. + * + * So i915 supports Flat-CCS, on the objects which can reside only on + * I915_MEMORY_CLASS_DEVICE regions. */ struct drm_i915_gem_create_ext_memory_regions { /** @base: Extension link. See struct i915_user_extension. */ -- GitLab From fae88d8791b2903771c85958903ee81080b66aec Mon Sep 17 00:00:00 2001 From: Lionel Landwerlin Date: Mon, 2 May 2022 12:38:16 +0300 Subject: [PATCH 2/5] anv: make use of the new smallbar uAPI Instead of having 2 VkMemoryType pointing to the same VkMemoryHeap, we have each VkMemoryType with VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT (one host visible, the other not) point to its own VkMemoryHeap. For the local heap that is host visible, we'll use the I915_GEM_CREATE_EXT_FLAG_NEEDS_CPU_ACCESS flag at GEM BO creation. When the smallbar uAPI is not available we fallback to a single heap and do not use I915_GEM_CREATE_EXT_FLAG_NEEDS_CPU_ACCESS. v2: Handle probed_cpu_visible_size == probed_size (Matthew) v3: * Jordan: Use region info from devinfo v4: Also make the vram host visible heap as local (Ken) Signed-off-by: Lionel Landwerlin Signed-off-by: Jordan Justen Reviewed-by: Kenneth Graunke Part-of: --- src/intel/vulkan/anv_allocator.c | 18 +++++---- src/intel/vulkan/anv_device.c | 64 ++++++++++++++++++++++++++------ src/intel/vulkan/anv_gem.c | 6 ++- src/intel/vulkan/anv_gem_stubs.c | 2 +- src/intel/vulkan/anv_private.h | 17 ++++++++- 5 files changed, 84 insertions(+), 23 deletions(-) diff --git a/src/intel/vulkan/anv_allocator.c b/src/intel/vulkan/anv_allocator.c index 652c20d8c791..f6aeb5cb5cca 100644 --- a/src/intel/vulkan/anv_allocator.c +++ b/src/intel/vulkan/anv_allocator.c @@ -1696,23 +1696,27 @@ anv_device_alloc_bo(struct anv_device *device, /* If we have vram size, we have multiple memory regions and should choose * one of them. */ - if (device->physical->vram.size > 0) { + if (anv_physical_device_has_vram(device->physical)) { struct drm_i915_gem_memory_class_instance regions[2]; uint32_t nregions = 0; if (alloc_flags & ANV_BO_ALLOC_LOCAL_MEM) { - regions[nregions++] = device->physical->vram.region; + /* vram_non_mappable & vram_mappable actually are the same region. */ + regions[nregions++] = device->physical->vram_non_mappable.region; } else { regions[nregions++] = device->physical->sys.region; } - /* TODO: Add I915_GEM_CREATE_EXT_FLAG_NEEDS_CPU_ACCESS to flags for - * after small BAR uapi is stabilized. - */ - assert(intel_vram_all_mappable(&device->info)); + uint32_t flags = 0; + if (alloc_flags & ANV_BO_ALLOC_LOCAL_MEM_CPU_VISIBLE) { + assert(alloc_flags & ANV_BO_ALLOC_LOCAL_MEM); + /* We're required to add smem as a region when using mappable vram. */ + regions[nregions++] = device->physical->sys.region; + flags |= I915_GEM_CREATE_EXT_FLAG_NEEDS_CPU_ACCESS; + } gem_handle = anv_gem_create_regions(device, size + ccs_size, - nregions, regions); + flags, nregions, regions); } else { gem_handle = anv_gem_create(device, size + ccs_size); } diff --git a/src/intel/vulkan/anv_device.c b/src/intel/vulkan/anv_device.c index 03cbea51f02e..63309be61ddc 100644 --- a/src/intel/vulkan/anv_device.c +++ b/src/intel/vulkan/anv_device.c @@ -378,11 +378,18 @@ anv_init_meminfo(struct anv_physical_device *device, int fd) anv_compute_sys_heap_size(device, devinfo->mem.sram.mappable.size); device->sys.available = devinfo->mem.sram.mappable.free; - device->vram.region.memory_class = devinfo->mem.vram.mem_class; - device->vram.region.memory_instance = + device->vram_mappable.region.memory_class = devinfo->mem.vram.mem_class; + device->vram_mappable.region.memory_instance = devinfo->mem.vram.mem_instance; - device->vram.size = devinfo->mem.vram.mappable.size; - device->vram.available = devinfo->mem.vram.mappable.free; + device->vram_mappable.size = devinfo->mem.vram.mappable.size; + device->vram_mappable.available = devinfo->mem.vram.mappable.free; + + device->vram_non_mappable.region.memory_class = + devinfo->mem.vram.mem_class; + device->vram_non_mappable.region.memory_instance = + devinfo->mem.vram.mem_instance; + device->vram_non_mappable.size = devinfo->mem.vram.unmappable.size; + device->vram_non_mappable.available = devinfo->mem.vram.unmappable.free; return VK_SUCCESS; } @@ -395,7 +402,8 @@ anv_update_meminfo(struct anv_physical_device *device, int fd) const struct intel_device_info *devinfo = &device->info; device->sys.available = devinfo->mem.sram.mappable.free; - device->vram.available = devinfo->mem.vram.mappable.free; + device->vram_mappable.available = devinfo->mem.vram.mappable.free; + device->vram_non_mappable.available = devinfo->mem.vram.unmappable.free; } @@ -408,13 +416,19 @@ anv_physical_device_init_heaps(struct anv_physical_device *device, int fd) assert(device->sys.size != 0); - if (device->vram.size > 0) { - /* We can create 2 different heaps when we have local memory support, - * first heap with local memory size and second with system memory size. + if (anv_physical_device_has_vram(device)) { + /* We can create 2 or 3 different heaps when we have local memory + * support, first heap with local memory size and second with system + * memory size and the third is added only if part of the vram is + * mappable to the host. */ device->memory.heap_count = 2; device->memory.heaps[0] = (struct anv_memory_heap) { - .size = device->vram.size, + /* If there is a vram_non_mappable, use that for the device only + * heap. Otherwise use the vram_mappable. + */ + .size = device->vram_non_mappable.size != 0 ? + device->vram_non_mappable.size : device->vram_mappable.size, .flags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT, .is_local_mem = true, }; @@ -423,6 +437,17 @@ anv_physical_device_init_heaps(struct anv_physical_device *device, int fd) .flags = 0, .is_local_mem = false, }; + /* Add an additional smaller vram mappable heap if we can't map all the + * vram to the host. + */ + if (device->vram_non_mappable.size > 0) { + device->memory.heap_count++; + device->memory.heaps[2] = (struct anv_memory_heap) { + .size = device->vram_mappable.size, + .flags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT, + .is_local_mem = true, + }; + } device->memory.type_count = 3; device->memory.types[0] = (struct anv_memory_type) { @@ -439,7 +464,11 @@ anv_physical_device_init_heaps(struct anv_physical_device *device, int fd) .propertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT | VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT, - .heapIndex = 0, + /* This memory type either comes from heaps[0] if there is only + * mappable vram region, or from heaps[2] if there is both mappable & + * non-mappable vram regions. + */ + .heapIndex = device->vram_non_mappable.size > 0 ? 2 : 0, }; } else if (device->info.has_llc) { device->memory.heap_count = 1; @@ -843,7 +872,8 @@ anv_physical_device_try_create(struct anv_instance *instance, device->gtt_size > (4ULL << 30 /* GiB */); /* Initialize memory regions struct to 0. */ - memset(&device->vram, 0, sizeof(device->vram)); + memset(&device->vram_non_mappable, 0, sizeof(device->vram_non_mappable)); + memset(&device->vram_mappable, 0, sizeof(device->vram_mappable)); memset(&device->sys, 0, sizeof(device->sys)); result = anv_physical_device_init_heaps(device, fd); @@ -2741,7 +2771,7 @@ anv_get_memory_budget(VkPhysicalDevice physicalDevice, if (device->memory.heaps[i].is_local_mem) { total_heaps_size = total_vram_heaps_size; - mem_available = device->vram.available; + mem_available = device->vram_non_mappable.available; } else { total_heaps_size = total_sys_heaps_size; mem_available = device->sys.available; @@ -3832,6 +3862,16 @@ VkResult anv_AllocateMemory( if (device->physical->has_implicit_ccs && device->info.has_aux_map) alloc_flags |= ANV_BO_ALLOC_IMPLICIT_CCS; + /* If i915 reported a mappable/non_mappable vram regions and the + * application want lmem mappable, then we need to use the + * I915_GEM_CREATE_EXT_FLAG_NEEDS_CPU_ACCESS flag to create our BO. + */ + if (pdevice->vram_mappable.size > 0 && + pdevice->vram_non_mappable.size > 0 && + (mem_type->propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) && + (mem_type->propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT)) + alloc_flags |= ANV_BO_ALLOC_LOCAL_MEM_CPU_VISIBLE; + if (vk_flags & VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT) alloc_flags |= ANV_BO_ALLOC_CLIENT_VISIBLE_ADDRESS; diff --git a/src/intel/vulkan/anv_gem.c b/src/intel/vulkan/anv_gem.c index 852b94cb957f..61896b402a26 100644 --- a/src/intel/vulkan/anv_gem.c +++ b/src/intel/vulkan/anv_gem.c @@ -66,9 +66,12 @@ anv_gem_close(struct anv_device *device, uint32_t gem_handle) uint32_t anv_gem_create_regions(struct anv_device *device, uint64_t anv_bo_size, - uint32_t num_regions, + uint32_t flags, uint32_t num_regions, struct drm_i915_gem_memory_class_instance *regions) { + /* Check for invalid flags */ + assert((flags & ~I915_GEM_CREATE_EXT_FLAG_NEEDS_CPU_ACCESS) == 0); + struct drm_i915_gem_create_ext_memory_regions ext_regions = { .base = { .name = I915_GEM_CREATE_EXT_MEMORY_REGIONS }, .num_regions = num_regions, @@ -78,6 +81,7 @@ anv_gem_create_regions(struct anv_device *device, uint64_t anv_bo_size, struct drm_i915_gem_create_ext gem_create = { .size = anv_bo_size, .extensions = (uintptr_t) &ext_regions, + .flags = flags, }; int ret = intel_ioctl(device->fd, DRM_IOCTL_I915_GEM_CREATE_EXT, diff --git a/src/intel/vulkan/anv_gem_stubs.c b/src/intel/vulkan/anv_gem_stubs.c index b42d8bc8bf1a..0a0f4c6e479b 100644 --- a/src/intel/vulkan/anv_gem_stubs.c +++ b/src/intel/vulkan/anv_gem_stubs.c @@ -47,7 +47,7 @@ anv_gem_close(struct anv_device *device, uint32_t gem_handle) uint32_t anv_gem_create_regions(struct anv_device *device, uint64_t anv_bo_size, - uint32_t num_regions, + uint32_t flags, uint32_t num_regions, struct drm_i915_gem_memory_class_instance *regions) { return 0; diff --git a/src/intel/vulkan/anv_private.h b/src/intel/vulkan/anv_private.h index a00ac6229423..5cfce5326228 100644 --- a/src/intel/vulkan/anv_private.h +++ b/src/intel/vulkan/anv_private.h @@ -1040,7 +1040,11 @@ struct anv_physical_device { bool need_clflush; } memory; - struct anv_memregion vram; + /* Either we have a single vram region and it's all mappable, or we have + * both mappable & non-mappable parts. System memory is always available. + */ + struct anv_memregion vram_mappable; + struct anv_memregion vram_non_mappable; struct anv_memregion sys; uint8_t driver_build_sha1[20]; uint8_t pipeline_cache_uuid[VK_UUID_SIZE]; @@ -1066,6 +1070,12 @@ struct anv_physical_device { struct intel_measure_device measure_device; }; +static inline bool +anv_physical_device_has_vram(const struct anv_physical_device *device) +{ + return device->vram_mappable.size > 0; +} + struct anv_app_info { const char* app_name; uint32_t app_version; @@ -1368,6 +1378,9 @@ enum anv_bo_alloc_flags { /** This buffer is allocated from local memory */ ANV_BO_ALLOC_LOCAL_MEM = (1 << 10), + + /** This buffer is allocated from local memory and should be cpu visible */ + ANV_BO_ALLOC_LOCAL_MEM_CPU_VISIBLE = (1 << 11), }; VkResult anv_device_alloc_bo(struct anv_device *device, @@ -1431,7 +1444,7 @@ void anv_gem_munmap(struct anv_device *device, void *p, uint64_t size); uint32_t anv_gem_create(struct anv_device *device, uint64_t size); void anv_gem_close(struct anv_device *device, uint32_t gem_handle); uint32_t anv_gem_create_regions(struct anv_device *device, uint64_t anv_bo_size, - uint32_t num_regions, + uint32_t flags, uint32_t num_regions, struct drm_i915_gem_memory_class_instance *regions); uint32_t anv_gem_userptr(struct anv_device *device, void *mem, size_t size); int anv_gem_busy(struct anv_device *device, uint32_t gem_handle); -- GitLab From ab4939dfff1d09b651ea6b72fba1ad0ac559120e Mon Sep 17 00:00:00 2001 From: Jordan Justen Date: Mon, 16 May 2022 10:43:15 -0700 Subject: [PATCH 3/5] iris/bufmgr: Add I915_GEM_CREATE_EXT_FLAG_NEEDS_CPU_ACCESS for vram mappable buffers Signed-off-by: Jordan Justen Reviewed-by: Kenneth Graunke Part-of: --- src/gallium/drivers/iris/iris_bufmgr.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/gallium/drivers/iris/iris_bufmgr.c b/src/gallium/drivers/iris/iris_bufmgr.c index 897f333346e0..9ad8d723ff5b 100644 --- a/src/gallium/drivers/iris/iris_bufmgr.c +++ b/src/gallium/drivers/iris/iris_bufmgr.c @@ -997,11 +997,10 @@ alloc_fresh_bo(struct iris_bufmgr *bufmgr, uint64_t bo_size, unsigned flags) .extensions = (uintptr_t)&ext_regions, }; - /* TODO: Add I915_GEM_CREATE_EXT_FLAG_NEEDS_CPU_ACCESS to create.flags - * for IRIS_HEAP_DEVICE_LOCAL_PREFERRED when small BAR uapi is - * stabilized. - */ - assert(bufmgr->all_vram_mappable); + if (!bufmgr->all_vram_mappable && + bo->real.heap == IRIS_HEAP_DEVICE_LOCAL_PREFERRED) { + create.flags |= I915_GEM_CREATE_EXT_FLAG_NEEDS_CPU_ACCESS; + } /* It should be safe to use GEM_CREATE_EXT without checking, since we are * in the side of the branch where discrete memory is available. So we -- GitLab From acc6457ff46a65f1a2c7f835b3050f55b0e663de Mon Sep 17 00:00:00 2001 From: Jordan Justen Date: Mon, 16 May 2022 02:33:11 -0700 Subject: [PATCH 4/5] intel/dev: Use i915 region probed_cpu_visible_size when non-zero Signed-off-by: Jordan Justen Reviewed-by: Lionel Landwerlin Reviewed-by: Kenneth Graunke Part-of: --- src/intel/dev/intel_device_info.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/intel/dev/intel_device_info.c b/src/intel/dev/intel_device_info.c index 28263c1e6dfd..29cbb849cad9 100644 --- a/src/intel/dev/intel_device_info.c +++ b/src/intel/dev/intel_device_info.c @@ -1625,7 +1625,18 @@ query_regions(struct intel_device_info *devinfo, int fd, bool update) if (!update) { devinfo->mem.vram.mem_class = mem->region.memory_class; devinfo->mem.vram.mem_instance = mem->region.memory_instance; - devinfo->mem.vram.mappable.size = mem->probed_size; + if (mem->probed_cpu_visible_size > 0) { + devinfo->mem.vram.mappable.size = mem->probed_cpu_visible_size; + devinfo->mem.vram.unmappable.size = + mem->probed_size - mem->probed_cpu_visible_size; + } else { + /* We are running on an older kernel without support for the + * small-bar uapi. These kernels only support systems where the + * entire vram is mappable. + */ + devinfo->mem.vram.mappable.size = mem->probed_size; + devinfo->mem.vram.unmappable.size = 0; + } } else { assert(devinfo->mem.vram.mem_class == mem->region.memory_class); assert(devinfo->mem.vram.mem_instance == mem->region.memory_instance); -- GitLab From 2863e720f03fccbba6258e54c10e129ddc8a5c12 Mon Sep 17 00:00:00 2001 From: Jordan Justen Date: Fri, 10 Jun 2022 18:20:30 -0700 Subject: [PATCH 5/5] intel/dev: Determine the amount of free vram using small BAR uapi Signed-off-by: Jordan Justen Reviewed-by: Lionel Landwerlin Reviewed-by: Kenneth Graunke Part-of: --- src/intel/dev/intel_device_info.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/intel/dev/intel_device_info.c b/src/intel/dev/intel_device_info.c index 29cbb849cad9..1e70ff5f7cff 100644 --- a/src/intel/dev/intel_device_info.c +++ b/src/intel/dev/intel_device_info.c @@ -1643,8 +1643,22 @@ query_regions(struct intel_device_info *devinfo, int fd, bool update) assert((devinfo->mem.vram.mappable.size + devinfo->mem.vram.unmappable.size) == mem->probed_size); } - if (mem->unallocated_size != -1) - devinfo->mem.vram.mappable.free = mem->unallocated_size; + if (mem->unallocated_cpu_visible_size > 0) { + if (mem->unallocated_size != -1) { + devinfo->mem.vram.mappable.free = mem->unallocated_cpu_visible_size; + devinfo->mem.vram.unmappable.free = + mem->unallocated_size - mem->unallocated_cpu_visible_size; + } + } else { + /* We are running on an older kernel without support for the + * small-bar uapi. These kernels only support systems where the + * entire vram is mappable. + */ + if (mem->unallocated_size != -1) { + devinfo->mem.vram.mappable.free = mem->unallocated_size; + devinfo->mem.vram.unmappable.free = 0; + } + } break; default: break; -- GitLab