radv_image.c 87.1 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
/*
 * Copyright © 2016 Red Hat.
 * Copyright © 2016 Bas Nieuwenhuizen
 *
 * based in part on anv driver which is:
 * Copyright © 2015 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

James Park's avatar
James Park committed
28
#include "ac_drm_fourcc.h"
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
29
30
31
#include "util/debug.h"
#include "util/u_atomic.h"
#include "vulkan/util/vk_format.h"
32
#include "radv_debug.h"
33
34
35
#include "radv_private.h"
#include "radv_radeon_winsys.h"
#include "sid.h"
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
36
37
#include "vk_format.h"
#include "vk_util.h"
38
39
40

#include "gfx10_format_table.h"

41
static const VkImageUsageFlagBits RADV_IMAGE_USAGE_WRITE_BITS =
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
42
43
   VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT |
   VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT | VK_IMAGE_USAGE_STORAGE_BIT;
44

45
static unsigned
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
46
47
radv_choose_tiling(struct radv_device *device, const VkImageCreateInfo *pCreateInfo,
                   VkFormat format)
48
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
   if (pCreateInfo->tiling == VK_IMAGE_TILING_LINEAR) {
      assert(pCreateInfo->samples <= 1);
      return RADEON_SURF_MODE_LINEAR_ALIGNED;
   }

   /* MSAA resources must be 2D tiled. */
   if (pCreateInfo->samples > 1)
      return RADEON_SURF_MODE_2D;

   if (!vk_format_is_compressed(format) && !vk_format_is_depth_or_stencil(format) &&
       device->physical_device->rad_info.chip_class <= GFX8) {
      /* this causes hangs in some VK CTS tests on GFX9. */
      /* Textures with a very small height are recommended to be linear. */
      if (pCreateInfo->imageType == VK_IMAGE_TYPE_1D ||
          /* Only very thin and long 2D textures should benefit from
           * linear_aligned. */
          (pCreateInfo->extent.width > 8 && pCreateInfo->extent.height <= 2))
         return RADEON_SURF_MODE_LINEAR_ALIGNED;
   }

   return RADEON_SURF_MODE_2D;
70
}
71
72

static bool
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
73
74
radv_use_tc_compat_htile_for_image(struct radv_device *device, const VkImageCreateInfo *pCreateInfo,
                                   VkFormat format)
75
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
   /* TC-compat HTILE is only available for GFX8+. */
   if (device->physical_device->rad_info.chip_class < GFX8)
      return false;

   if ((pCreateInfo->usage & VK_IMAGE_USAGE_STORAGE_BIT))
      return false;

   if (pCreateInfo->tiling == VK_IMAGE_TILING_LINEAR)
      return false;

   /* Do not enable TC-compatible HTILE if the image isn't readable by a
    * shader because no texture fetches will happen.
    */
   if (!(pCreateInfo->usage & (VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT |
                               VK_IMAGE_USAGE_TRANSFER_SRC_BIT)))
      return false;

   if (device->physical_device->rad_info.chip_class < GFX9) {
      /* TC-compat HTILE for MSAA depth/stencil images is broken
       * on GFX8 because the tiling doesn't match.
       */
      if (pCreateInfo->samples >= 2 && format == VK_FORMAT_D32_SFLOAT_S8_UINT)
         return false;

      /* GFX9+ supports compression for both 32-bit and 16-bit depth
       * surfaces, while GFX8 only supports 32-bit natively. Though,
       * the driver allows TC-compat HTILE for 16-bit depth surfaces
       * with no Z planes compression.
       */
      if (format != VK_FORMAT_D32_SFLOAT_S8_UINT && format != VK_FORMAT_D32_SFLOAT &&
          format != VK_FORMAT_D16_UNORM)
         return false;
   }

   return true;
111
112
}

113
114
115
static bool
radv_surface_has_scanout(struct radv_device *device, const struct radv_image_create_info *info)
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
116
117
118
119
120
121
122
123
   if (info->bo_metadata) {
      if (device->physical_device->rad_info.chip_class >= GFX9)
         return info->bo_metadata->u.gfx9.scanout;
      else
         return info->bo_metadata->u.legacy.scanout;
   }

   return info->scanout;
124
125
}

126
static bool
127
128
radv_image_use_fast_clear_for_image(const struct radv_device *device,
                                    const struct radv_image *image)
129
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
   if (device->instance->debug_flags & RADV_DEBUG_FORCE_COMPRESS)
      return true;

   if (image->info.samples <= 1 && image->info.width * image->info.height <= 512 * 512) {
      /* Do not enable CMASK or DCC for small surfaces where the cost
       * of the eliminate pass can be higher than the benefit of fast
       * clear. RadeonSI does this, but the image threshold is
       * different.
       */
      return false;
   }

   return image->usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT &&
          (image->exclusive ||
           /* Enable DCC for concurrent images if stores are
            * supported because that means we can keep DCC compressed on
            * all layouts/queues.
            */
           radv_image_use_dcc_image_stores(device, image));
149
150
}

151
bool
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
152
153
radv_are_formats_dcc_compatible(const struct radv_physical_device *pdev, const void *pNext,
                                VkFormat format, VkImageCreateFlags flags)
154
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
   bool blendable;

   if (!radv_is_colorbuffer_format_supported(pdev, format, &blendable))
      return false;

   if (flags & VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT) {
      const struct VkImageFormatListCreateInfo *format_list =
         (const struct VkImageFormatListCreateInfo *)vk_find_struct_const(
            pNext, IMAGE_FORMAT_LIST_CREATE_INFO);

      /* We have to ignore the existence of the list if viewFormatCount = 0 */
      if (format_list && format_list->viewFormatCount) {
         /* compatibility is transitive, so we only need to check
          * one format with everything else. */
         for (unsigned i = 0; i < format_list->viewFormatCount; ++i) {
            if (format_list->pViewFormats[i] == VK_FORMAT_UNDEFINED)
               continue;

            if (!radv_dcc_formats_compatible(format, format_list->pViewFormats[i]))
               return false;
         }
      } else {
         return false;
      }
   }

   return true;
182
183
}

184
static bool
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
185
radv_formats_is_atomic_allowed(const void *pNext, VkFormat format, VkImageCreateFlags flags)
186
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
   if (radv_is_atomic_format_supported(format))
      return true;

   if (flags & VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT) {
      const struct VkImageFormatListCreateInfo *format_list =
         (const struct VkImageFormatListCreateInfo *)vk_find_struct_const(
            pNext, IMAGE_FORMAT_LIST_CREATE_INFO);

      /* We have to ignore the existence of the list if viewFormatCount = 0 */
      if (format_list && format_list->viewFormatCount) {
         for (unsigned i = 0; i < format_list->viewFormatCount; ++i) {
            if (radv_is_atomic_format_supported(format_list->pViewFormats[i]))
               return true;
         }
      }
   }

   return false;
205
206
}

207
static bool
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
208
209
radv_use_dcc_for_image(struct radv_device *device, const struct radv_image *image,
                       const VkImageCreateInfo *pCreateInfo, VkFormat format)
210
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
   /* DCC (Delta Color Compression) is only available for GFX8+. */
   if (device->physical_device->rad_info.chip_class < GFX8)
      return false;

   if (device->instance->debug_flags & RADV_DEBUG_NO_DCC)
      return false;

   if (image->shareable && image->tiling != VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT)
      return false;

   /*
    * TODO: Enable DCC for storage images on GFX9 and earlier.
    *
    * Also disable DCC with atomics because even when DCC stores are
    * supported atomics will always decompress. So if we are
    * decompressing a lot anyway we might as well not have DCC.
    */
   if ((pCreateInfo->usage & VK_IMAGE_USAGE_STORAGE_BIT) &&
       (!radv_image_use_dcc_image_stores(device, image) ||
        radv_formats_is_atomic_allowed(pCreateInfo->pNext, format, pCreateInfo->flags)))
      return false;

233
234
235
236
   /* Do not enable DCC for fragment shading rate attachments. */
   if (pCreateInfo->usage & VK_IMAGE_USAGE_FRAGMENT_SHADING_RATE_ATTACHMENT_BIT_KHR)
      return false;

Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
   if (pCreateInfo->tiling == VK_IMAGE_TILING_LINEAR)
      return false;

   if (vk_format_is_subsampled(format) || vk_format_get_plane_count(format) > 1)
      return false;

   if (!radv_image_use_fast_clear_for_image(device, image) &&
       image->tiling != VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT)
      return false;

   /* Do not enable DCC for mipmapped arrays because performance is worse. */
   if (pCreateInfo->arrayLayers > 1 && pCreateInfo->mipLevels > 1)
      return false;

   if (device->physical_device->rad_info.chip_class < GFX10) {
      /* TODO: Add support for DCC MSAA on GFX8-9. */
      if (pCreateInfo->samples > 1 && !device->physical_device->dcc_msaa_allowed)
         return false;

      /* TODO: Add support for DCC layers/mipmaps on GFX9. */
      if ((pCreateInfo->arrayLayers > 1 || pCreateInfo->mipLevels > 1) &&
          device->physical_device->rad_info.chip_class == GFX9)
         return false;
   }

   return radv_are_formats_dcc_compatible(device->physical_device, pCreateInfo->pNext, format,
                                          pCreateInfo->flags);
264
265
}

266
267
268
269
270
271
272
273
274
275
/*
 * Whether to enable image stores with DCC compression for this image. If
 * this function returns false the image subresource should be decompressed
 * before using it with image stores.
 *
 * Note that this can have mixed performance implications, see
 * https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/6796#note_643299
 *
 * This function assumes the image uses DCC compression.
 */
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
276
277
bool
radv_image_use_dcc_image_stores(const struct radv_device *device, const struct radv_image *image)
278
{
279
   return device->physical_device->rad_info.chip_class >= GFX10;
280
281
282
283
284
285
}

/*
 * Whether to use a predicate to determine whether DCC is in a compressed
 * state. This can be used to avoid decompressing an image multiple times.
 */
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
286
287
bool
radv_image_use_dcc_predication(const struct radv_device *device, const struct radv_image *image)
288
{
289
   return radv_image_has_dcc(image) && !radv_image_use_dcc_image_stores(device, image);
290
291
}

292
static inline bool
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
293
radv_use_fmask_for_image(const struct radv_device *device, const struct radv_image *image)
294
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
295
296
   return image->info.samples > 1 && ((image->usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) ||
                                      (device->instance->debug_flags & RADV_DEBUG_FORCE_COMPRESS));
297
298
}

299
static inline bool
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
300
radv_use_htile_for_image(const struct radv_device *device, const struct radv_image *image)
301
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
302
303
304
305
306
307
308
   /* TODO:
    * - Investigate about mips+layers.
    * - Enable on other gens.
    */
   bool use_htile_for_mips =
      image->info.array_size == 1 && device->physical_device->rad_info.chip_class >= GFX10;

309
310
311
312
313
314
315
316
317
   /* Do not enable HTILE for very small images because it seems less performant but make sure it's
    * allowed with VRS attachments because we need HTILE.
    */
   if (image->info.width * image->info.height < 8 * 8 &&
       !(device->instance->debug_flags & RADV_DEBUG_FORCE_COMPRESS) &&
       !device->attachment_vrs_enabled)
      return false;

   return (image->info.levels == 1 || use_htile_for_mips) && !image->shareable;
318
319
}

320
static bool
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
321
radv_use_tc_compat_cmask_for_image(struct radv_device *device, struct radv_image *image)
322
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
   /* TC-compat CMASK is only available for GFX8+. */
   if (device->physical_device->rad_info.chip_class < GFX8)
      return false;

   if (device->instance->debug_flags & RADV_DEBUG_NO_TC_COMPAT_CMASK)
      return false;

   if (image->usage & VK_IMAGE_USAGE_STORAGE_BIT)
      return false;

   /* Do not enable TC-compatible if the image isn't readable by a shader
    * because no texture fetches will happen.
    */
   if (!(image->usage & (VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT |
                         VK_IMAGE_USAGE_TRANSFER_SRC_BIT)))
      return false;

   /* If the image doesn't have FMASK, it can't be fetchable. */
   if (!radv_image_has_fmask(image))
      return false;

   return true;
345
346
}

Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
347
348
static uint32_t
si_get_bo_metadata_word1(const struct radv_device *device)
349
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
350
   return (ATI_VENDOR_ID << 16) | device->physical_device->rad_info.pci_id;
351
352
353
}

static bool
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
354
radv_is_valid_opaque_metadata(const struct radv_device *device, const struct radeon_bo_metadata *md)
355
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
356
357
   if (md->metadata[0] != 1 || md->metadata[1] != si_get_bo_metadata_word1(device))
      return false;
358

Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
359
360
   if (md->size_metadata < 40)
      return false;
361

Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
362
   return true;
363
364
}

365
static void
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
366
radv_patch_surface_from_metadata(struct radv_device *device, struct radeon_surf *surface,
367
                                 const struct radeon_bo_metadata *md)
368
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
369
370
371
372
373
374
375
376
   surface->flags = RADEON_SURF_CLR(surface->flags, MODE);

   if (device->physical_device->rad_info.chip_class >= GFX9) {
      if (md->u.gfx9.swizzle_mode > 0)
         surface->flags |= RADEON_SURF_SET(RADEON_SURF_MODE_2D, MODE);
      else
         surface->flags |= RADEON_SURF_SET(RADEON_SURF_MODE_LINEAR_ALIGNED, MODE);

377
      surface->u.gfx9.swizzle_mode = md->u.gfx9.swizzle_mode;
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
   } else {
      surface->u.legacy.pipe_config = md->u.legacy.pipe_config;
      surface->u.legacy.bankw = md->u.legacy.bankw;
      surface->u.legacy.bankh = md->u.legacy.bankh;
      surface->u.legacy.tile_split = md->u.legacy.tile_split;
      surface->u.legacy.mtilea = md->u.legacy.mtilea;
      surface->u.legacy.num_banks = md->u.legacy.num_banks;

      if (md->u.legacy.macrotile == RADEON_LAYOUT_TILED)
         surface->flags |= RADEON_SURF_SET(RADEON_SURF_MODE_2D, MODE);
      else if (md->u.legacy.microtile == RADEON_LAYOUT_TILED)
         surface->flags |= RADEON_SURF_SET(RADEON_SURF_MODE_1D, MODE);
      else
         surface->flags |= RADEON_SURF_SET(RADEON_SURF_MODE_LINEAR_ALIGNED, MODE);
   }
393
394
}

395
static VkResult
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
396
radv_patch_image_dimensions(struct radv_device *device, struct radv_image *image,
397
398
399
                            const struct radv_image_create_info *create_info,
                            struct ac_surf_info *image_info)
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
   unsigned width = image->info.width;
   unsigned height = image->info.height;

   /*
    * minigbm sometimes allocates bigger images which is going to result in
    * weird strides and other properties. Lets be lenient where possible and
    * fail it on GFX10 (as we cannot cope there).
    *
    * Example hack: https://chromium-review.googlesource.com/c/chromiumos/platform/minigbm/+/1457777/
    */
   if (create_info->bo_metadata &&
       radv_is_valid_opaque_metadata(device, create_info->bo_metadata)) {
      const struct radeon_bo_metadata *md = create_info->bo_metadata;

      if (device->physical_device->rad_info.chip_class >= GFX10) {
         width = G_00A004_WIDTH_LO(md->metadata[3]) + (G_00A008_WIDTH_HI(md->metadata[4]) << 2) + 1;
         height = G_00A008_HEIGHT(md->metadata[4]) + 1;
      } else {
         width = G_008F18_WIDTH(md->metadata[4]) + 1;
         height = G_008F18_HEIGHT(md->metadata[4]) + 1;
      }
   }

   if (image->info.width == width && image->info.height == height)
      return VK_SUCCESS;

   if (width < image->info.width || height < image->info.height) {
      fprintf(stderr,
              "The imported image has smaller dimensions than the internal\n"
              "dimensions. Using it is going to fail badly, so we reject\n"
              "this import.\n"
              "(internal dimensions: %d x %d, external dimensions: %d x %d)\n",
              image->info.width, image->info.height, width, height);
      return VK_ERROR_INVALID_EXTERNAL_HANDLE;
   } else if (device->physical_device->rad_info.chip_class >= GFX10) {
      fprintf(stderr,
              "Tried to import an image with inconsistent width on GFX10.\n"
              "As GFX10 has no separate stride fields we cannot cope with\n"
              "an inconsistency in width and will fail this import.\n"
              "(internal dimensions: %d x %d, external dimensions: %d x %d)\n",
              image->info.width, image->info.height, width, height);
      return VK_ERROR_INVALID_EXTERNAL_HANDLE;
   } else {
      fprintf(stderr,
              "Tried to import an image with inconsistent width on pre-GFX10.\n"
              "As GFX10 has no separate stride fields we cannot cope with\n"
              "an inconsistency and would fail on GFX10.\n"
              "(internal dimensions: %d x %d, external dimensions: %d x %d)\n",
              image->info.width, image->info.height, width, height);
   }
   image_info->width = width;
   image_info->height = height;

   return VK_SUCCESS;
454
455
456
}

static VkResult
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
457
radv_patch_image_from_extra_info(struct radv_device *device, struct radv_image *image,
458
459
                                 const struct radv_image_create_info *create_info,
                                 struct ac_surf_info *image_info)
460
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
   VkResult result = radv_patch_image_dimensions(device, image, create_info, image_info);
   if (result != VK_SUCCESS)
      return result;

   for (unsigned plane = 0; plane < image->plane_count; ++plane) {
      if (create_info->bo_metadata) {
         radv_patch_surface_from_metadata(device, &image->planes[plane].surface,
                                          create_info->bo_metadata);
      }

      if (radv_surface_has_scanout(device, create_info)) {
         image->planes[plane].surface.flags |= RADEON_SURF_SCANOUT;
         if (device->instance->debug_flags & RADV_DEBUG_NO_DISPLAY_DCC)
            image->planes[plane].surface.flags |= RADEON_SURF_DISABLE_DCC;

         image->info.surf_index = NULL;
      }
   }
   return VK_SUCCESS;
480
481
}

482
static uint64_t
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
483
484
radv_get_surface_flags(struct radv_device *device, const struct radv_image *image,
                       unsigned plane_id, const VkImageCreateInfo *pCreateInfo,
485
                       VkFormat image_format)
486
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
   uint64_t flags;
   unsigned array_mode = radv_choose_tiling(device, pCreateInfo, image_format);
   VkFormat format = vk_format_get_plane_format(image_format, plane_id);
   const struct util_format_description *desc = vk_format_description(format);
   bool is_depth, is_stencil;

   is_depth = util_format_has_depth(desc);
   is_stencil = util_format_has_stencil(desc);

   flags = RADEON_SURF_SET(array_mode, MODE);

   switch (pCreateInfo->imageType) {
   case VK_IMAGE_TYPE_1D:
      if (pCreateInfo->arrayLayers > 1)
         flags |= RADEON_SURF_SET(RADEON_SURF_TYPE_1D_ARRAY, TYPE);
      else
         flags |= RADEON_SURF_SET(RADEON_SURF_TYPE_1D, TYPE);
      break;
   case VK_IMAGE_TYPE_2D:
      if (pCreateInfo->arrayLayers > 1)
         flags |= RADEON_SURF_SET(RADEON_SURF_TYPE_2D_ARRAY, TYPE);
      else
         flags |= RADEON_SURF_SET(RADEON_SURF_TYPE_2D, TYPE);
      break;
   case VK_IMAGE_TYPE_3D:
      flags |= RADEON_SURF_SET(RADEON_SURF_TYPE_3D, TYPE);
      break;
   default:
      unreachable("unhandled image type");
   }

   /* Required for clearing/initializing a specific layer on GFX8. */
   flags |= RADEON_SURF_CONTIGUOUS_DCC_LAYERS;

   if (is_depth) {
      flags |= RADEON_SURF_ZBUFFER;

      if (radv_use_htile_for_image(device, image) &&
          !(device->instance->debug_flags & RADV_DEBUG_NO_HIZ)) {
         if (radv_use_tc_compat_htile_for_image(device, pCreateInfo, image_format))
            flags |= RADEON_SURF_TC_COMPATIBLE_HTILE;
      } else {
         flags |= RADEON_SURF_NO_HTILE;
      }
   }

   if (is_stencil)
      flags |= RADEON_SURF_SBUFFER;

   if (device->physical_device->rad_info.chip_class >= GFX9 &&
       pCreateInfo->imageType == VK_IMAGE_TYPE_3D &&
       vk_format_get_blocksizebits(image_format) == 128 && vk_format_is_compressed(image_format))
      flags |= RADEON_SURF_NO_RENDER_TARGET;

   if (!radv_use_dcc_for_image(device, image, pCreateInfo, image_format))
      flags |= RADEON_SURF_DISABLE_DCC;

   if (!radv_use_fmask_for_image(device, image))
      flags |= RADEON_SURF_NO_FMASK;

   if (pCreateInfo->flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT) {
      flags |=
         RADEON_SURF_PRT | RADEON_SURF_NO_FMASK | RADEON_SURF_NO_HTILE | RADEON_SURF_DISABLE_DCC;
   }

   return flags;
553
}
554

555
static inline unsigned
556
si_tile_mode_index(const struct radv_image_plane *plane, unsigned level, bool stencil)
557
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
558
   if (stencil)
559
      return plane->surface.u.legacy.zs.stencil_tiling_index[level];
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
560
561
   else
      return plane->surface.u.legacy.tiling_index[level];
562
563
}

Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
564
565
static unsigned
radv_map_swizzle(unsigned swizzle)
566
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
567
568
569
570
571
572
573
574
575
576
577
578
579
580
   switch (swizzle) {
   case PIPE_SWIZZLE_Y:
      return V_008F0C_SQ_SEL_Y;
   case PIPE_SWIZZLE_Z:
      return V_008F0C_SQ_SEL_Z;
   case PIPE_SWIZZLE_W:
      return V_008F0C_SQ_SEL_W;
   case PIPE_SWIZZLE_0:
      return V_008F0C_SQ_SEL_0;
   case PIPE_SWIZZLE_1:
      return V_008F0C_SQ_SEL_1;
   default: /* PIPE_SWIZZLE_X */
      return V_008F0C_SQ_SEL_X;
   }
581
582
}

583
static void
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
584
585
radv_compose_swizzle(const struct util_format_description *desc, const VkComponentMapping *mapping,
                     enum pipe_swizzle swizzle[4])
586
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
   if (desc->format == PIPE_FORMAT_R64_UINT || desc->format == PIPE_FORMAT_R64_SINT) {
      /* 64-bit formats only support storage images and storage images
       * require identity component mappings. We use 32-bit
       * instructions to access 64-bit images, so we need a special
       * case here.
       *
       * The zw components are 1,0 so that they can be easily be used
       * by loads to create the w component, which has to be 0 for
       * NULL descriptors.
       */
      swizzle[0] = PIPE_SWIZZLE_X;
      swizzle[1] = PIPE_SWIZZLE_Y;
      swizzle[2] = PIPE_SWIZZLE_1;
      swizzle[3] = PIPE_SWIZZLE_0;
   } else if (!mapping) {
      for (unsigned i = 0; i < 4; i++)
         swizzle[i] = desc->swizzle[i];
   } else if (desc->colorspace == UTIL_FORMAT_COLORSPACE_ZS) {
      const unsigned char swizzle_xxxx[4] = {PIPE_SWIZZLE_X, PIPE_SWIZZLE_0, PIPE_SWIZZLE_0,
                                             PIPE_SWIZZLE_1};
      vk_format_compose_swizzles(mapping, swizzle_xxxx, swizzle);
   } else {
      vk_format_compose_swizzles(mapping, desc->swizzle, swizzle);
   }
611
612
}

613
static void
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
614
615
radv_make_buffer_descriptor(struct radv_device *device, struct radv_buffer *buffer,
                            VkFormat vk_format, unsigned offset, unsigned range, uint32_t *state)
616
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
   const struct util_format_description *desc;
   unsigned stride;
   uint64_t gpu_address = radv_buffer_get_va(buffer->bo);
   uint64_t va = gpu_address + buffer->offset;
   unsigned num_format, data_format;
   int first_non_void;
   enum pipe_swizzle swizzle[4];
   desc = vk_format_description(vk_format);
   first_non_void = vk_format_get_first_non_void_channel(vk_format);
   stride = desc->block.bits / 8;

   radv_compose_swizzle(desc, NULL, swizzle);

   va += offset;
   state[0] = va;
   state[1] = S_008F04_BASE_ADDRESS_HI(va >> 32) | S_008F04_STRIDE(stride);

   if (device->physical_device->rad_info.chip_class != GFX8 && stride) {
      range /= stride;
   }

   state[2] = range;
   state[3] = S_008F0C_DST_SEL_X(radv_map_swizzle(swizzle[0])) |
              S_008F0C_DST_SEL_Y(radv_map_swizzle(swizzle[1])) |
              S_008F0C_DST_SEL_Z(radv_map_swizzle(swizzle[2])) |
              S_008F0C_DST_SEL_W(radv_map_swizzle(swizzle[3]));

   if (device->physical_device->rad_info.chip_class >= GFX10) {
      const struct gfx10_format *fmt = &gfx10_format_table[vk_format_to_pipe_format(vk_format)];

      /* OOB_SELECT chooses the out-of-bounds check:
       *  - 0: (index >= NUM_RECORDS) || (offset >= STRIDE)
       *  - 1: index >= NUM_RECORDS
       *  - 2: NUM_RECORDS == 0
       *  - 3: if SWIZZLE_ENABLE == 0: offset >= NUM_RECORDS
       *       else: swizzle_address >= NUM_RECORDS
       */
      state[3] |= S_008F0C_FORMAT(fmt->img_format) |
                  S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_STRUCTURED_WITH_OFFSET) |
                  S_008F0C_RESOURCE_LEVEL(1);
   } else {
      num_format = radv_translate_buffer_numformat(desc, first_non_void);
      data_format = radv_translate_buffer_dataformat(desc, first_non_void);

      assert(data_format != V_008F0C_BUF_DATA_FORMAT_INVALID);
      assert(num_format != ~0);

      state[3] |= S_008F0C_NUM_FORMAT(num_format) | S_008F0C_DATA_FORMAT(data_format);
   }
666
667
668
}

static void
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
669
670
671
672
673
si_set_mutable_tex_desc_fields(struct radv_device *device, struct radv_image *image,
                               const struct legacy_surf_level *base_level_info, unsigned plane_id,
                               unsigned base_level, unsigned first_level, unsigned block_width,
                               bool is_stencil, bool is_storage_image, bool disable_compression,
                               bool enable_write_compression, uint32_t *state)
674
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
675
676
677
678
679
680
681
   struct radv_image_plane *plane = &image->planes[plane_id];
   uint64_t gpu_address = image->bo ? radv_buffer_get_va(image->bo) + image->offset : 0;
   uint64_t va = gpu_address;
   enum chip_class chip_class = device->physical_device->rad_info.chip_class;
   uint64_t meta_va = 0;
   if (chip_class >= GFX9) {
      if (is_stencil)
682
         va += plane->surface.u.gfx9.zs.stencil_offset;
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
683
684
685
      else
         va += plane->surface.u.gfx9.surf_offset;
   } else
686
      va += (uint64_t)base_level_info->offset_256B * 256;
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
687
688
689
690
691
692
693
694
695
696
697

   state[0] = va >> 8;
   if (chip_class >= GFX9 || base_level_info->mode == RADEON_SURF_MODE_2D)
      state[0] |= plane->surface.tile_swizzle;
   state[1] &= C_008F14_BASE_ADDRESS_HI;
   state[1] |= S_008F14_BASE_ADDRESS_HI(va >> 40);

   if (chip_class >= GFX8) {
      state[6] &= C_008F28_COMPRESSION_EN;
      state[7] = 0;
      if (!disable_compression && radv_dcc_enabled(image, first_level)) {
698
         meta_va = gpu_address + plane->surface.meta_offset;
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
699
         if (chip_class <= GFX8)
700
            meta_va += plane->surface.u.legacy.color.dcc_level[base_level].dcc_offset;
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
701
702

         unsigned dcc_tile_swizzle = plane->surface.tile_swizzle << 8;
703
         dcc_tile_swizzle &= (1 << plane->surface.meta_alignment_log2) - 1;
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
704
705
         meta_va |= dcc_tile_swizzle;
      } else if (!disable_compression && radv_image_is_tc_compat_htile(image)) {
706
         meta_va = gpu_address + plane->surface.meta_offset;
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
707
708
709
710
711
712
713
714
715
716
717
718
719
      }

      if (meta_va) {
         state[6] |= S_008F28_COMPRESSION_EN(1);
         if (chip_class <= GFX9)
            state[7] = meta_va >> 8;
      }
   }

   if (chip_class >= GFX10) {
      state[3] &= C_00A00C_SW_MODE;

      if (is_stencil) {
720
         state[3] |= S_00A00C_SW_MODE(plane->surface.u.gfx9.zs.stencil_swizzle_mode);
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
721
      } else {
722
         state[3] |= S_00A00C_SW_MODE(plane->surface.u.gfx9.swizzle_mode);
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
723
724
725
726
727
728
729
730
731
732
      }

      state[6] &= C_00A018_META_DATA_ADDRESS_LO & C_00A018_META_PIPE_ALIGNED;

      if (meta_va) {
         struct gfx9_surf_meta_flags meta = {
            .rb_aligned = 1,
            .pipe_aligned = 1,
         };

733
         if (!(plane->surface.flags & RADEON_SURF_Z_OR_SBUFFER))
734
            meta = plane->surface.u.gfx9.color.dcc;
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
735
736
737
738
739
740
741
742
743
744
745
746
747
748

         if (radv_dcc_enabled(image, first_level) && is_storage_image && enable_write_compression)
            state[6] |= S_00A018_WRITE_COMPRESS_ENABLE(1);

         state[6] |= S_00A018_META_PIPE_ALIGNED(meta.pipe_aligned) |
                     S_00A018_META_DATA_ADDRESS_LO(meta_va >> 8);
      }

      state[7] = meta_va >> 16;
   } else if (chip_class == GFX9) {
      state[3] &= C_008F1C_SW_MODE;
      state[4] &= C_008F20_PITCH;

      if (is_stencil) {
749
750
         state[3] |= S_008F1C_SW_MODE(plane->surface.u.gfx9.zs.stencil_swizzle_mode);
         state[4] |= S_008F20_PITCH(plane->surface.u.gfx9.zs.stencil_epitch);
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
751
      } else {
752
753
         state[3] |= S_008F1C_SW_MODE(plane->surface.u.gfx9.swizzle_mode);
         state[4] |= S_008F20_PITCH(plane->surface.u.gfx9.epitch);
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
754
755
756
757
758
759
760
761
762
763
      }

      state[5] &=
         C_008F24_META_DATA_ADDRESS & C_008F24_META_PIPE_ALIGNED & C_008F24_META_RB_ALIGNED;
      if (meta_va) {
         struct gfx9_surf_meta_flags meta = {
            .rb_aligned = 1,
            .pipe_aligned = 1,
         };

764
         if (!(plane->surface.flags & RADEON_SURF_Z_OR_SBUFFER))
765
            meta = plane->surface.u.gfx9.color.dcc;
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780

         state[5] |= S_008F24_META_DATA_ADDRESS(meta_va >> 40) |
                     S_008F24_META_PIPE_ALIGNED(meta.pipe_aligned) |
                     S_008F24_META_RB_ALIGNED(meta.rb_aligned);
      }
   } else {
      /* GFX6-GFX8 */
      unsigned pitch = base_level_info->nblk_x * block_width;
      unsigned index = si_tile_mode_index(plane, base_level, is_stencil);

      state[3] &= C_008F1C_TILING_INDEX;
      state[3] |= S_008F1C_TILING_INDEX(index);
      state[4] &= C_008F20_PITCH;
      state[4] |= S_008F20_PITCH(pitch - 1);
   }
781
782
}

Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
783
784
785
static unsigned
radv_tex_dim(VkImageType image_type, VkImageViewType view_type, unsigned nr_layers,
             unsigned nr_samples, bool is_storage_image, bool gfx9)
786
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
   if (view_type == VK_IMAGE_VIEW_TYPE_CUBE || view_type == VK_IMAGE_VIEW_TYPE_CUBE_ARRAY)
      return is_storage_image ? V_008F1C_SQ_RSRC_IMG_2D_ARRAY : V_008F1C_SQ_RSRC_IMG_CUBE;

   /* GFX9 allocates 1D textures as 2D. */
   if (gfx9 && image_type == VK_IMAGE_TYPE_1D)
      image_type = VK_IMAGE_TYPE_2D;
   switch (image_type) {
   case VK_IMAGE_TYPE_1D:
      return nr_layers > 1 ? V_008F1C_SQ_RSRC_IMG_1D_ARRAY : V_008F1C_SQ_RSRC_IMG_1D;
   case VK_IMAGE_TYPE_2D:
      if (nr_samples > 1)
         return nr_layers > 1 ? V_008F1C_SQ_RSRC_IMG_2D_MSAA_ARRAY : V_008F1C_SQ_RSRC_IMG_2D_MSAA;
      else
         return nr_layers > 1 ? V_008F1C_SQ_RSRC_IMG_2D_ARRAY : V_008F1C_SQ_RSRC_IMG_2D;
   case VK_IMAGE_TYPE_3D:
      if (view_type == VK_IMAGE_VIEW_TYPE_3D)
         return V_008F1C_SQ_RSRC_IMG_3D;
      else
         return V_008F1C_SQ_RSRC_IMG_2D_ARRAY;
   default:
      unreachable("illegal image type");
   }
809
}
810

Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
811
static unsigned
812
gfx9_border_color_swizzle(const struct util_format_description *desc)
813
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
814
815
   unsigned bc_swizzle = V_008F20_BC_SWIZZLE_XYZW;

816
   if (desc->swizzle[3] == PIPE_SWIZZLE_X) {
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
817
818
819
820
821
822
      /* For the pre-defined border color values (white, opaque
       * black, transparent black), the only thing that matters is
       * that the alpha channel winds up in the correct place
       * (because the RGB channels are all the same) so either of
       * these enumerations will work.
       */
823
      if (desc->swizzle[2] == PIPE_SWIZZLE_Y)
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
824
825
826
         bc_swizzle = V_008F20_BC_SWIZZLE_WZYX;
      else
         bc_swizzle = V_008F20_BC_SWIZZLE_WXYZ;
827
828
   } else if (desc->swizzle[0] == PIPE_SWIZZLE_X) {
      if (desc->swizzle[1] == PIPE_SWIZZLE_Y)
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
829
830
831
         bc_swizzle = V_008F20_BC_SWIZZLE_XYZW;
      else
         bc_swizzle = V_008F20_BC_SWIZZLE_XWYZ;
832
   } else if (desc->swizzle[1] == PIPE_SWIZZLE_X) {
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
833
      bc_swizzle = V_008F20_BC_SWIZZLE_YXWZ;
834
   } else if (desc->swizzle[2] == PIPE_SWIZZLE_X) {
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
835
836
837
838
      bc_swizzle = V_008F20_BC_SWIZZLE_ZYXW;
   }

   return bc_swizzle;
839
840
}

Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
841
842
bool
vi_alpha_is_on_msb(struct radv_device *device, VkFormat format)
843
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
844
   const struct util_format_description *desc = vk_format_description(format);
845

Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
846
847
   if (device->physical_device->rad_info.chip_class >= GFX10 && desc->nr_channels == 1)
      return desc->swizzle[3] == PIPE_SWIZZLE_X;
848

Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
849
   return radv_translate_colorswap(format, false) <= 1;
850
}
851
/**
852
853
854
 * Build the sampler view descriptor for a texture (GFX10).
 */
static void
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
855
856
857
858
859
860
gfx10_make_texture_descriptor(struct radv_device *device, struct radv_image *image,
                              bool is_storage_image, VkImageViewType view_type, VkFormat vk_format,
                              const VkComponentMapping *mapping, unsigned first_level,
                              unsigned last_level, unsigned first_layer, unsigned last_layer,
                              unsigned width, unsigned height, unsigned depth, uint32_t *state,
                              uint32_t *fmask_state)
861
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
   const struct util_format_description *desc;
   enum pipe_swizzle swizzle[4];
   unsigned img_format;
   unsigned type;

   desc = vk_format_description(vk_format);
   img_format = gfx10_format_table[vk_format_to_pipe_format(vk_format)].img_format;

   radv_compose_swizzle(desc, mapping, swizzle);

   type = radv_tex_dim(image->type, view_type, image->info.array_size, image->info.samples,
                       is_storage_image, device->physical_device->rad_info.chip_class == GFX9);
   if (type == V_008F1C_SQ_RSRC_IMG_1D_ARRAY) {
      height = 1;
      depth = image->info.array_size;
   } else if (type == V_008F1C_SQ_RSRC_IMG_2D_ARRAY || type == V_008F1C_SQ_RSRC_IMG_2D_MSAA_ARRAY) {
      if (view_type != VK_IMAGE_VIEW_TYPE_3D)
         depth = image->info.array_size;
   } else if (type == V_008F1C_SQ_RSRC_IMG_CUBE)
      depth = image->info.array_size / 6;

   state[0] = 0;
   state[1] = S_00A004_FORMAT(img_format) | S_00A004_WIDTH_LO(width - 1);
   state[2] = S_00A008_WIDTH_HI((width - 1) >> 2) | S_00A008_HEIGHT(height - 1) |
              S_00A008_RESOURCE_LEVEL(1);
   state[3] = S_00A00C_DST_SEL_X(radv_map_swizzle(swizzle[0])) |
              S_00A00C_DST_SEL_Y(radv_map_swizzle(swizzle[1])) |
              S_00A00C_DST_SEL_Z(radv_map_swizzle(swizzle[2])) |
              S_00A00C_DST_SEL_W(radv_map_swizzle(swizzle[3])) |
              S_00A00C_BASE_LEVEL(image->info.samples > 1 ? 0 : first_level) |
              S_00A00C_LAST_LEVEL(image->info.samples > 1 ? util_logbase2(image->info.samples)
                                                          : last_level) |
894
              S_00A00C_BC_SWIZZLE(gfx9_border_color_swizzle(desc)) | S_00A00C_TYPE(type);
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
   /* Depth is the the last accessible layer on gfx9+. The hw doesn't need
    * to know the total number of layers.
    */
   state[4] = S_00A010_DEPTH(type == V_008F1C_SQ_RSRC_IMG_3D ? depth - 1 : last_layer) |
              S_00A010_BASE_ARRAY(first_layer);
   state[5] = S_00A014_ARRAY_PITCH(0) |
              S_00A014_MAX_MIP(image->info.samples > 1 ? util_logbase2(image->info.samples)
                                                       : image->info.levels - 1) |
              S_00A014_PERF_MOD(4);
   state[6] = 0;
   state[7] = 0;

   if (radv_dcc_enabled(image, first_level)) {
      state[6] |= S_00A018_MAX_UNCOMPRESSED_BLOCK_SIZE(V_028C78_MAX_BLOCK_SIZE_256B) |
                  S_00A018_MAX_COMPRESSED_BLOCK_SIZE(
910
                     image->planes[0].surface.u.gfx9.color.dcc.max_compressed_block_size) |
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
911
912
913
                  S_00A018_ALPHA_IS_ON_MSB(vi_alpha_is_on_msb(device, vk_format));
   }

914
915
916
917
   if (radv_image_get_iterate256(device, image)) {
      state[6] |= S_00A018_ITERATE_256(1);
   }

Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
918
919
920
921
922
923
924
925
926
927
928
929
930
   /* Initialize the sampler view for FMASK. */
   if (fmask_state) {
      if (radv_image_has_fmask(image)) {
         uint64_t gpu_address = radv_buffer_get_va(image->bo);
         uint32_t format;
         uint64_t va;

         assert(image->plane_count == 1);

         va = gpu_address + image->offset + image->planes[0].surface.fmask_offset;

         switch (image->info.samples) {
         case 2:
931
            format = V_008F0C_GFX10_FORMAT_FMASK8_S2_F2;
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
932
933
            break;
         case 4:
934
            format = V_008F0C_GFX10_FORMAT_FMASK8_S4_F4;
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
935
936
            break;
         case 8:
937
            format = V_008F0C_GFX10_FORMAT_FMASK32_S8_F8;
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
938
939
940
941
942
943
944
945
946
947
948
949
950
            break;
         default:
            unreachable("invalid nr_samples");
         }

         fmask_state[0] = (va >> 8) | image->planes[0].surface.fmask_tile_swizzle;
         fmask_state[1] = S_00A004_BASE_ADDRESS_HI(va >> 40) | S_00A004_FORMAT(format) |
                          S_00A004_WIDTH_LO(width - 1);
         fmask_state[2] = S_00A008_WIDTH_HI((width - 1) >> 2) | S_00A008_HEIGHT(height - 1) |
                          S_00A008_RESOURCE_LEVEL(1);
         fmask_state[3] =
            S_00A00C_DST_SEL_X(V_008F1C_SQ_SEL_X) | S_00A00C_DST_SEL_Y(V_008F1C_SQ_SEL_X) |
            S_00A00C_DST_SEL_Z(V_008F1C_SQ_SEL_X) | S_00A00C_DST_SEL_W(V_008F1C_SQ_SEL_X) |
951
            S_00A00C_SW_MODE(image->planes[0].surface.u.gfx9.color.fmask_swizzle_mode) |
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
            S_00A00C_TYPE(
               radv_tex_dim(image->type, view_type, image->info.array_size, 0, false, false));
         fmask_state[4] = S_00A010_DEPTH(last_layer) | S_00A010_BASE_ARRAY(first_layer);
         fmask_state[5] = 0;
         fmask_state[6] = S_00A018_META_PIPE_ALIGNED(1);
         fmask_state[7] = 0;

         if (radv_image_is_tc_compat_cmask(image)) {
            va = gpu_address + image->offset + image->planes[0].surface.cmask_offset;

            fmask_state[6] |= S_00A018_COMPRESSION_EN(1);
            fmask_state[6] |= S_00A018_META_DATA_ADDRESS_LO(va >> 8);
            fmask_state[7] |= va >> 16;
         }
      } else
         memset(fmask_state, 0, 8 * 4);
   }
969
970
971
972
}

/**
 * Build the sampler view descriptor for a texture (SI-GFX9)
973
974
 */
static void
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
975
976
977
978
979
980
si_make_texture_descriptor(struct radv_device *device, struct radv_image *image,
                           bool is_storage_image, VkImageViewType view_type, VkFormat vk_format,
                           const VkComponentMapping *mapping, unsigned first_level,
                           unsigned last_level, unsigned first_layer, unsigned last_layer,
                           unsigned width, unsigned height, unsigned depth, uint32_t *state,
                           uint32_t *fmask_state)
981
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
   const struct util_format_description *desc;
   enum pipe_swizzle swizzle[4];
   int first_non_void;
   unsigned num_format, data_format, type;

   desc = vk_format_description(vk_format);

   radv_compose_swizzle(desc, mapping, swizzle);

   first_non_void = vk_format_get_first_non_void_channel(vk_format);

   num_format = radv_translate_tex_numformat(vk_format, desc, first_non_void);
   if (num_format == ~0) {
      num_format = 0;
   }

   data_format = radv_translate_tex_dataformat(vk_format, desc, first_non_void);
   if (data_format == ~0) {
      data_format = 0;
   }

   /* S8 with either Z16 or Z32 HTILE need a special format. */
   if (device->physical_device->rad_info.chip_class == GFX9 && vk_format == VK_FORMAT_S8_UINT &&
       radv_image_is_tc_compat_htile(image)) {
      if (image->vk_format == VK_FORMAT_D32_SFLOAT_S8_UINT)
         data_format = V_008F14_IMG_DATA_FORMAT_S8_32;
      else if (image->vk_format == VK_FORMAT_D16_UNORM_S8_UINT)
         data_format = V_008F14_IMG_DATA_FORMAT_S8_16;
   }
   type = radv_tex_dim(image->type, view_type, image->info.array_size, image->info.samples,
                       is_storage_image, device->physical_device->rad_info.chip_class == GFX9);
   if (type == V_008F1C_SQ_RSRC_IMG_1D_ARRAY) {
      height = 1;
      depth = image->info.array_size;
   } else if (type == V_008F1C_SQ_RSRC_IMG_2D_ARRAY || type == V_008F1C_SQ_RSRC_IMG_2D_MSAA_ARRAY) {
      if (view_type != VK_IMAGE_VIEW_TYPE_3D)
         depth = image->info.array_size;
   } else if (type == V_008F1C_SQ_RSRC_IMG_CUBE)
      depth = image->info.array_size / 6;

   state[0] = 0;
   state[1] = (S_008F14_DATA_FORMAT(data_format) | S_008F14_NUM_FORMAT(num_format));
   state[2] = (S_008F18_WIDTH(width - 1) | S_008F18_HEIGHT(height - 1) | S_008F18_PERF_MOD(4));
   state[3] = (S_008F1C_DST_SEL_X(radv_map_swizzle(swizzle[0])) |
               S_008F1C_DST_SEL_Y(radv_map_swizzle(swizzle[1])) |
               S_008F1C_DST_SEL_Z(radv_map_swizzle(swizzle[2])) |
               S_008F1C_DST_SEL_W(radv_map_swizzle(swizzle[3])) |
               S_008F1C_BASE_LEVEL(image->info.samples > 1 ? 0 : first_level) |
               S_008F1C_LAST_LEVEL(image->info.samples > 1 ? util_logbase2(image->info.samples)
                                                           : last_level) |
               S_008F1C_TYPE(type));
   state[4] = 0;
   state[5] = S_008F24_BASE_ARRAY(first_layer);
   state[6] = 0;
   state[7] = 0;

   if (device->physical_device->rad_info.chip_class == GFX9) {
1039
      unsigned bc_swizzle = gfx9_border_color_swizzle(desc);
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056

      /* Depth is the last accessible layer on Gfx9.
       * The hw doesn't need to know the total number of layers.
       */
      if (type == V_008F1C_SQ_RSRC_IMG_3D)
         state[4] |= S_008F20_DEPTH(depth - 1);
      else
         state[4] |= S_008F20_DEPTH(last_layer);

      state[4] |= S_008F20_BC_SWIZZLE(bc_swizzle);
      state[5] |= S_008F24_MAX_MIP(image->info.samples > 1 ? util_logbase2(image->info.samples)
                                                           : image->info.levels - 1);
   } else {
      state[3] |= S_008F1C_POW2_PAD(image->info.levels > 1);
      state[4] |= S_008F20_DEPTH(depth - 1);
      state[5] |= S_008F24_LAST_ARRAY(last_layer);
   }
1057
1058
   if (!(image->planes[0].surface.flags & RADEON_SURF_Z_OR_SBUFFER) &&
       image->planes[0].surface.meta_offset) {
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
      state[6] = S_008F28_ALPHA_IS_ON_MSB(vi_alpha_is_on_msb(device, vk_format));
   } else {
      /* The last dword is unused by hw. The shader uses it to clear
       * bits in the first dword of sampler state.
       */
      if (device->physical_device->rad_info.chip_class <= GFX7 && image->info.samples <= 1) {
         if (first_level == last_level)
            state[7] = C_008F30_MAX_ANISO_RATIO;
         else
            state[7] = 0xffffffff;
      }
   }

   /* Initialize the sampler view for FMASK. */
   if (fmask_state) {
      if (radv_image_has_fmask(image)) {
         uint32_t fmask_format;
         uint64_t gpu_address = radv_buffer_get_va(image->bo);
         uint64_t va;

         assert(image->plane_count == 1);

         va = gpu_address + image->offset + image->planes[0].surface.fmask_offset;

         if (device->physical_device->rad_info.chip_class == GFX9) {
            fmask_format = V_008F14_IMG_DATA_FORMAT_FMASK;
            switch (image->info.samples) {
            case 2:
               num_format = V_008F14_IMG_NUM_FORMAT_FMASK_8_2_2;
               break;
            case 4:
               num_format = V_008F14_IMG_NUM_FORMAT_FMASK_8_4_4;
               break;
            case 8:
               num_format = V_008F14_IMG_NUM_FORMAT_FMASK_32_8_8;
               break;
            default:
               unreachable("invalid nr_samples");
            }
         } else {
            switch (image->info.samples) {
            case 2:
               fmask_format = V_008F14_IMG_DATA_FORMAT_FMASK8_S2_F2;
               break;
            case 4:
               fmask_format = V_008F14_IMG_DATA_FORMAT_FMASK8_S4_F4;
               break;
            case 8:
               fmask_format = V_008F14_IMG_DATA_FORMAT_FMASK32_S8_F8;
               break;
            default:
               assert(0);
               fmask_format = V_008F14_IMG_DATA_FORMAT_INVALID;
            }
            num_format = V_008F14_IMG_NUM_FORMAT_UINT;
         }

         fmask_state[0] = va >> 8;
         fmask_state[0] |= image->planes[0].surface.fmask_tile_swizzle;
         fmask_state[1] = S_008F14_BASE_ADDRESS_HI(va >> 40) | S_008F14_DATA_FORMAT(fmask_format) |
                          S_008F14_NUM_FORMAT(num_format);
         fmask_state[2] = S_008F18_WIDTH(width - 1) | S_008F18_HEIGHT(height - 1);
         fmask_state[3] =
            S_008F1C_DST_SEL_X(V_008F1C_SQ_SEL_X) | S_008F1C_DST_SEL_Y(V_008F1C_SQ_SEL_X) |
            S_008F1C_DST_SEL_Z(V_008F1C_SQ_SEL_X) | S_008F1C_DST_SEL_W(V_008F1C_SQ_SEL_X) |
            S_008F1C_TYPE(
               radv_tex_dim(image->type, view_type, image->info.array_size, 0, false, false));
         fmask_state[4] = 0;
         fmask_state[5] = S_008F24_BASE_ARRAY(first_layer);
         fmask_state[6] = 0;
         fmask_state[7] = 0;

         if (device->physical_device->rad_info.chip_class == GFX9) {
1132
            fmask_state[3] |= S_008F1C_SW_MODE(image->planes[0].surface.u.gfx9.color.fmask_swizzle_mode);
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1133
            fmask_state[4] |= S_008F20_DEPTH(last_layer) |
1134
                              S_008F20_PITCH(image->planes[0].surface.u.gfx9.color.fmask_epitch);
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
            fmask_state[5] |= S_008F24_META_PIPE_ALIGNED(1) | S_008F24_META_RB_ALIGNED(1);

            if (radv_image_is_tc_compat_cmask(image)) {
               va = gpu_address + image->offset + image->planes[0].surface.cmask_offset;

               fmask_state[5] |= S_008F24_META_DATA_ADDRESS(va >> 40);
               fmask_state[6] |= S_008F28_COMPRESSION_EN(1);
               fmask_state[7] |= va >> 8;
            }
         } else {
            fmask_state[3] |=
1146
               S_008F1C_TILING_INDEX(image->planes[0].surface.u.legacy.color.fmask.tiling_index);
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1147
1148
            fmask_state[4] |=
               S_008F20_DEPTH(depth - 1) |
1149
               S_008F20_PITCH(image->planes[0].surface.u.legacy.color.fmask.pitch_in_pixels - 1);
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
            fmask_state[5] |= S_008F24_LAST_ARRAY(last_layer);

            if (radv_image_is_tc_compat_cmask(image)) {
               va = gpu_address + image->offset + image->planes[0].surface.cmask_offset;

               fmask_state[6] |= S_008F28_COMPRESSION_EN(1);
               fmask_state[7] |= va >> 8;
            }
         }
      } else
         memset(fmask_state, 0, 8 * 4);
   }
1162
1163
}

1164
static void
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1165
1166
1167
1168
1169
1170
radv_make_texture_descriptor(struct radv_device *device, struct radv_image *image,
                             bool is_storage_image, VkImageViewType view_type, VkFormat vk_format,
                             const VkComponentMapping *mapping, unsigned first_level,
                             unsigned last_level, unsigned first_layer, unsigned last_layer,
                             unsigned width, unsigned height, unsigned depth, uint32_t *state,
                             uint32_t *fmask_state)
1171
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1172
1173
1174
1175
1176
1177
1178
1179
1180
   if (device->physical_device->rad_info.chip_class >= GFX10) {
      gfx10_make_texture_descriptor(device, image, is_storage_image, view_type, vk_format, mapping,
                                    first_level, last_level, first_layer, last_layer, width, height,
                                    depth, state, fmask_state);
   } else {
      si_make_texture_descriptor(device, image, is_storage_image, view_type, vk_format, mapping,
                                 first_level, last_level, first_layer, last_layer, width, height,
                                 depth, state, fmask_state);
   }
1181
1182
}

1183
static void
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1184
1185
radv_query_opaque_metadata(struct radv_device *device, struct radv_image *image,
                           struct radeon_bo_metadata *md)
1186
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1187
1188
   static const VkComponentMapping fixedmapping;
   uint32_t desc[8];
1189

Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1190
   assert(image->plane_count == 1);
1191

Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1192
1193
1194
1195
   radv_make_texture_descriptor(device, image, false, (VkImageViewType)image->type,
                                image->vk_format, &fixedmapping, 0, image->info.levels - 1, 0,
                                image->info.array_size - 1, image->info.width, image->info.height,
                                image->info.depth, desc, NULL);
1196

Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1197
1198
1199
   si_set_mutable_tex_desc_fields(device, image, &image->planes[0].surface.u.legacy.level[0], 0, 0,
                                  0, image->planes[0].surface.blk_w, false, false, false, false,
                                  desc);
1200

Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1201
1202
   ac_surface_get_umd_metadata(&device->physical_device->rad_info, &image->planes[0].surface,
                               image->info.levels, desc, &md->size_metadata, md->metadata);
1203
1204
1205
}

void
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1206
1207
radv_init_metadata(struct radv_device *device, struct radv_image *image,
                   struct radeon_bo_metadata *metadata)
1208
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1209
1210
1211
1212
1213
1214
1215
   struct radeon_surf *surface = &image->planes[0].surface;

   memset(metadata, 0, sizeof(*metadata));

   if (device->physical_device->rad_info.chip_class >= GFX9) {
      uint64_t dcc_offset =
         image->offset +
1216
         (surface->display_dcc_offset ? surface->display_dcc_offset : surface->meta_offset);
1217
      metadata->u.gfx9.swizzle_mode = surface->u.gfx9.swizzle_mode;
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1218
      metadata->u.gfx9.dcc_offset_256b = dcc_offset >> 8;
1219
1220
1221
      metadata->u.gfx9.dcc_pitch_max = surface->u.gfx9.color.display_dcc_pitch_max;
      metadata->u.gfx9.dcc_independent_64b_blocks = surface->u.gfx9.color.dcc.independent_64B_blocks;
      metadata->u.gfx9.dcc_independent_128b_blocks = surface->u.gfx9.color.dcc.independent_128B_blocks;
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1222
      metadata->u.gfx9.dcc_max_compressed_block_size =
1223
         surface->u.gfx9.color.dcc.max_compressed_block_size;
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
      metadata->u.gfx9.scanout = (surface->flags & RADEON_SURF_SCANOUT) != 0;
   } else {
      metadata->u.legacy.microtile = surface->u.legacy.level[0].mode >= RADEON_SURF_MODE_1D
                                        ? RADEON_LAYOUT_TILED
                                        : RADEON_LAYOUT_LINEAR;
      metadata->u.legacy.macrotile = surface->u.legacy.level[0].mode >= RADEON_SURF_MODE_2D
                                        ? RADEON_LAYOUT_TILED
                                        : RADEON_LAYOUT_LINEAR;
      metadata->u.legacy.pipe_config = surface->u.legacy.pipe_config;
      metadata->u.legacy.bankw = surface->u.legacy.bankw;
      metadata->u.legacy.bankh = surface->u.legacy.bankh;
      metadata->u.legacy.tile_split = surface->u.legacy.tile_split;
      metadata->u.legacy.mtilea = surface->u.legacy.mtilea;
      metadata->u.legacy.num_banks = surface->u.legacy.num_banks;
      metadata->u.legacy.stride = surface->u.legacy.level[0].nblk_x * surface->bpe;
      metadata->u.legacy.scanout = (surface->flags & RADEON_SURF_SCANOUT) != 0;
   }
   radv_query_opaque_metadata(device, image, metadata);
1242
1243
}

1244
void
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1245
radv_image_override_offset_stride(struct radv_device *device, struct radv_image *image,
1246
1247
                                  uint64_t offset, uint32_t stride)
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1248
1249
   ac_surface_override_offset_stride(&device->physical_device->rad_info, &image->planes[0].surface,
                                     image->info.levels, offset, stride);
1250
1251
}

1252
static void
1253
radv_image_alloc_single_sample_cmask(const struct radv_device *device,
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1254
                                     const struct radv_image *image, struct radeon_surf *surf)
1255
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1256
1257
1258
1259
1260
1261
1262
1263
   if (!surf->cmask_size || surf->cmask_offset || surf->bpe > 8 || image->info.levels > 1 ||
       image->info.depth > 1 || radv_image_has_dcc(image) ||
       !radv_image_use_fast_clear_for_image(device, image) ||
       (image->flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT))
      return;

   assert(image->info.storage_samples == 1);

1264
   surf->cmask_offset = align64(surf->total_size, 1 << surf->cmask_alignment_log2);
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1265
   surf->total_size = surf->cmask_offset + surf->cmask_size;
1266
   surf->alignment_log2 = MAX2(surf->alignment_log2, surf->cmask_alignment_log2);
1267
1268
1269
1270
1271
}

static void
radv_image_alloc_values(const struct radv_device *device, struct radv_image *image)
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
   /* images with modifiers can be potentially imported */
   if (image->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT)
      return;

   if (radv_image_has_cmask(image) || radv_image_has_dcc(image)) {
      image->fce_pred_offset = image->size;
      image->size += 8 * image->info.levels;
   }

   if (radv_image_use_dcc_predication(device, image)) {
      image->dcc_pred_offset = image->size;
      image->size += 8 * image->info.levels;
   }

   if (radv_image_has_dcc(image) || radv_image_has_cmask(image) || radv_image_has_htile(image)) {
      image->clear_value_offset = image->size;
      image->size += 8 * image->info.levels;
   }

   if (radv_image_is_tc_compat_htile(image) &&
       device->physical_device->rad_info.has_tc_compat_zrange_bug) {
      /* Metadata for the TC-compatible HTILE hardware bug which
       * have to be fixed by updating ZRANGE_PRECISION when doing
       * fast depth clears to 0.0f.
       */
      image->tc_compat_zrange_offset = image->size;
      image->size += image->info.levels * 4;
   }
1300
1301
}

1302
1303
1304
static void
radv_image_reset_layout(struct radv_image *image)
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
   image->size = 0;
   image->alignment = 1;

   image->tc_compatible_cmask = 0;
   image->fce_pred_offset = image->dcc_pred_offset = 0;
   image->clear_value_offset = image->tc_compat_zrange_offset = 0;

   for (unsigned i = 0; i < image->plane_count; ++i) {
      VkFormat format = vk_format_get_plane_format(image->vk_format, i);

      uint64_t flags = image->planes[i].surface.flags;
      uint64_t modifier = image->planes[i].surface.modifier;
      memset(image->planes + i, 0, sizeof(image->planes[i]));

      image->planes[i].surface.flags = flags;
      image->planes[i].surface.modifier = modifier;
      image->planes[i].surface.blk_w = vk_format_get_blockwidth(format);
      image->planes[i].surface.blk_h = vk_format_get_blockheight(format);
      image->planes[i].surface.bpe = vk_format_get_blocksize(vk_format_depth_only(format));

      /* align byte per element on dword */
      if (image->planes[i].surface.bpe == 3) {
         image->planes[i].surface.bpe = 4;
      }
   }
1330
1331
}

1332
VkResult
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1333
radv_image_create_layout(struct radv_device *device, struct radv_image_create_info create_info,
1334
                         const struct VkImageDrmFormatModifierExplicitCreateInfoEXT *mod_info,
1335
                         struct radv_image *image)
1336
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
   /* Clear the pCreateInfo pointer so we catch issues in the delayed case when we test in the
    * common internal case. */
   create_info.vk_info = NULL;

   struct ac_surf_info image_info = image->info;
   VkResult result = radv_patch_image_from_extra_info(device, image, &create_info, &image_info);
   if (result != VK_SUCCESS)
      return result;

   assert(!mod_info || mod_info->drmFormatModifierPlaneCount >= image->plane_count);

   radv_image_reset_layout(image);

   for (unsigned plane = 0; plane < image->plane_count; ++plane) {
      struct ac_surf_info info = image_info;
      uint64_t offset;
      unsigned stride;

      info.width = vk_format_get_plane_width(image->vk_format, plane, info.width);
      info.height = vk_format_get_plane_height(image->vk_format, plane, info.height);

      if (create_info.no_metadata_planes || image->plane_count > 1) {
         image->planes[plane].surface.flags |=
            RADEON_SURF_DISABLE_DCC | RADEON_SURF_NO_FMASK | RADEON_SURF_NO_HTILE;
      }

      device->ws->surface_init(device->ws, &info, &image->planes[plane].surface);

      if (create_info.bo_metadata && !mod_info &&
          !ac_surface_set_umd_metadata(&device->physical_device->rad_info,
                                       &image->planes[plane].surface, image_info.storage_samples,
                                       image_info.levels, create_info.bo_metadata->size_metadata,
                                       create_info.bo_metadata->metadata))
         return VK_ERROR_INVALID_EXTERNAL_HANDLE;

      if (!create_info.no_metadata_planes && !create_info.bo_metadata && image->plane_count == 1 &&
          !mod_info)
         radv_image_alloc_single_sample_cmask(device, image, &image->planes[plane].surface);

      if (mod_info) {
         if (mod_info->pPlaneLayouts[plane].rowPitch % image->planes[plane].surface.bpe ||
             !mod_info->pPlaneLayouts[plane].rowPitch)
            return VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT;

         offset = mod_info->pPlaneLayouts[plane].offset;
         stride = mod_info->pPlaneLayouts[plane].rowPitch / image->planes[plane].surface.bpe;
      } else {
1384
         offset = align64(image->size, 1 << image->planes[plane].surface.alignment_log2);
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
         stride = 0; /* 0 means no override */
      }

      if (!ac_surface_override_offset_stride(&device->physical_device->rad_info,
                                             &image->planes[plane].surface, image->info.levels,
                                             offset, stride))
         return VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT;

      /* Validate DCC offsets in modifier layout. */
      if (image->plane_count == 1 && mod_info) {
         unsigned mem_planes = ac_surface_get_nplanes(&image->planes[plane].surface);
         if (mod_info->drmFormatModifierPlaneCount != mem_planes)
            return VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT;

         for (unsigned i = 1; i < mem_planes; ++i) {
            if (ac_surface_get_plane_offset(device->physical_device->rad_info.chip_class,
                                            &image->planes[plane].surface, i,
                                            0) != mod_info->pPlaneLayouts[i].offset)
               return VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT;
         }
      }

      image->size = MAX2(image->size, offset + image->planes[plane].surface.total_size);
1408
      image->alignment = MAX2(image->alignment, 1 << image->planes[plane].surface.alignment_log2);
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421

      image->planes[plane].format = vk_format_get_plane_format(image->vk_format, plane);
   }

   image->tc_compatible_cmask =
      radv_image_has_cmask(image) && radv_use_tc_compat_cmask_for_image(device, image);

   radv_image_alloc_values(device, image);

   assert(image->planes[0].surface.surf_size);
   assert(image->planes[0].surface.modifier == DRM_FORMAT_MOD_INVALID ||
          ac_modifier_has_dcc(image->planes[0].surface.modifier) == radv_image_has_dcc(image));
   return VK_SUCCESS;
1422
1423
}

1424
static void
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1425
1426
radv_destroy_image(struct radv_device *device, const VkAllocationCallbacks *pAllocator,
                   struct radv_image *image)
1427
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1428
1429
   if ((image->flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT) && image->bo)
      device->ws->buffer_destroy(device->ws, image->bo);
1430

Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1431
1432
1433
1434
   if (image->owned_memory != VK_NULL_HANDLE) {
      RADV_FROM_HANDLE(radv_device_memory, mem, image->owned_memory);
      radv_free_memory(device, pAllocator, mem);
   }
1435

Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1436
1437
   vk_object_base_finish(&image->base);
   vk_free2(&device->vk.alloc, pAllocator, image);
1438
1439
}

Simon Ser's avatar
Simon Ser committed
1440
1441
1442
static void
radv_image_print_info(struct radv_device *device, struct radv_image *image)
{
Bas Nieuwenhuizen's avatar
Bas Nieuwenhuizen committed
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
   fprintf(stderr, "Image:\n");
   fprintf(stderr,
           "  Info: size=%" PRIu64 ", alignment=%" PRIu32 ", "
           "width=%" PRIu32 ", height=%" PRIu32 ", "
           "offset=%" PRIu64 ", array_size=%" PRIu32 "\n",
           image->size, image->alignment, image->info.width, image->info.height, image->offset,
           image->info.array_size);
   for (unsigned i = 0; i < image->plane_count; ++i) {
      const struct radv_image_plane *plane = &image->planes[i];
      const struct radeon_surf *surf = &plane->surface;
      const struct util_format_description *desc = vk_format_description(plane->format);
      uint64_t offset = ac_surface_get_plane_offset(device->physical_device->rad_info.chip_class,
                                                    &plane->surface, 0, 0);

      fprintf(stderr, "  Plane[%u]: vkformat=%s, offset=%" PRIu64 "\n", i, desc->name, offset);

      ac_surface_print_info(stderr, &device->physical_device->rad_info, surf);
   }
Simon Ser's avatar
Simon Ser committed
1461
1462
}

1463
1464
1465
1466
/**
 * Determine if the given image can be fast cleared.
 */
static bool
Bas Nieuwenhuizen's avatar