freedreno_resource.c 51.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
/*
 * Copyright (C) 2012 Rob Clark <robclark@freedesktop.org>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 * Authors:
 *    Rob Clark <robclark@freedesktop.org>
 */

27
28
29
#include "util/format/u_format.h"
#include "util/format/u_format_rgtc.h"
#include "util/format/u_format_zs.h"
Rob Clark's avatar
Rob Clark committed
30
31
#include "util/set.h"
#include "util/u_drm.h"
32
33
#include "util/u_inlines.h"
#include "util/u_string.h"
Rob Clark's avatar
Rob Clark committed
34
#include "util/u_surface.h"
Rob Clark's avatar
Rob Clark committed
35
#include "util/u_transfer.h"
36

37
38
#include "decode/util.h"

39
#include "freedreno_batch_cache.h"
40
#include "freedreno_blitter.h"
Rob Clark's avatar
Rob Clark committed
41
#include "freedreno_context.h"
42
#include "freedreno_fence.h"
Rob Clark's avatar
Rob Clark committed
43
44
#include "freedreno_query_hw.h"
#include "freedreno_resource.h"
45
46
47
48
#include "freedreno_screen.h"
#include "freedreno_surface.h"
#include "freedreno_util.h"

49
#include <errno.h>
Rob Clark's avatar
Rob Clark committed
50
#include "drm-uapi/drm_fourcc.h"
51

Rob Clark's avatar
Rob Clark committed
52
/* XXX this should go away, needed for 'struct winsys_handle' */
53
#include "frontend/drm_driver.h"
54

55
56
57
58
59
60
/* A private modifier for now, so we have a way to request tiled but not
 * compressed.  It would perhaps be good to get real modifiers for the
 * tiled formats, but would probably need to do some work to figure out
 * the layout(s) of the tiled modes, and whether they are the same
 * across generations.
 */
Rob Clark's avatar
Rob Clark committed
61
#define FD_FORMAT_MOD_QCOM_TILED fourcc_mod_code(QCOM, 0xffffffff)
62

63
64
65
/**
 * Go through the entire state and see if the resource is bound
 * anywhere. If it is, mark the relevant state as dirty. This is
66
 * called on realloc_bo to ensure the necessary state is re-
67
68
 * emitted so the GPU looks at the new backing bo.
 */
69
static void
Rob Clark's avatar
Rob Clark committed
70
71
rebind_resource_in_ctx(struct fd_context *ctx,
                       struct fd_resource *rsc) assert_dt
72
{
Rob Clark's avatar
Rob Clark committed
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
   struct pipe_resource *prsc = &rsc->b.b;

   if (ctx->rebind_resource)
      ctx->rebind_resource(ctx, rsc);

   /* VBOs */
   if (rsc->dirty & FD_DIRTY_VTXBUF) {
      struct fd_vertexbuf_stateobj *vb = &ctx->vtx.vertexbuf;
      for (unsigned i = 0; i < vb->count && !(ctx->dirty & FD_DIRTY_VTXBUF);
           i++) {
         if (vb->vb[i].buffer.resource == prsc)
            fd_context_dirty(ctx, FD_DIRTY_VTXBUF);
      }
   }

   const enum fd_dirty_3d_state per_stage_dirty =
      FD_DIRTY_CONST | FD_DIRTY_TEX | FD_DIRTY_IMAGE | FD_DIRTY_SSBO;

   if (!(rsc->dirty & per_stage_dirty))
      return;

   /* per-shader-stage resources: */
   for (unsigned stage = 0; stage < PIPE_SHADER_TYPES; stage++) {
      /* Constbufs.. note that constbuf[0] is normal uniforms emitted in
       * cmdstream rather than by pointer..
       */
      if ((rsc->dirty & FD_DIRTY_CONST) &&
          !(ctx->dirty_shader[stage] & FD_DIRTY_CONST)) {
         struct fd_constbuf_stateobj *cb = &ctx->constbuf[stage];
         const unsigned num_ubos = util_last_bit(cb->enabled_mask);
         for (unsigned i = 1; i < num_ubos; i++) {
            if (cb->cb[i].buffer == prsc) {
               fd_context_dirty_shader(ctx, stage, FD_DIRTY_SHADER_CONST);
               break;
            }
         }
      }

      /* Textures */
      if ((rsc->dirty & FD_DIRTY_TEX) &&
          !(ctx->dirty_shader[stage] & FD_DIRTY_TEX)) {
         struct fd_texture_stateobj *tex = &ctx->tex[stage];
         for (unsigned i = 0; i < tex->num_textures; i++) {
            if (tex->textures[i] && (tex->textures[i]->texture == prsc)) {
               fd_context_dirty_shader(ctx, stage, FD_DIRTY_SHADER_TEX);
               break;
            }
         }
      }

      /* Images */
      if ((rsc->dirty & FD_DIRTY_IMAGE) &&
          !(ctx->dirty_shader[stage] & FD_DIRTY_IMAGE)) {
         struct fd_shaderimg_stateobj *si = &ctx->shaderimg[stage];
         const unsigned num_images = util_last_bit(si->enabled_mask);
         for (unsigned i = 0; i < num_images; i++) {
            if (si->si[i].resource == prsc) {
               fd_context_dirty_shader(ctx, stage, FD_DIRTY_SHADER_IMAGE);
               break;
            }
         }
      }

      /* SSBOs */
      if ((rsc->dirty & FD_DIRTY_SSBO) &&
          !(ctx->dirty_shader[stage] & FD_DIRTY_SSBO)) {
         struct fd_shaderbuf_stateobj *sb = &ctx->shaderbuf[stage];
         const unsigned num_ssbos = util_last_bit(sb->enabled_mask);
         for (unsigned i = 0; i < num_ssbos; i++) {
            if (sb->sb[i].buffer == prsc) {
               fd_context_dirty_shader(ctx, stage, FD_DIRTY_SHADER_SSBO);
               break;
            }
         }
      }
   }
149
150
}

151
static void
Rob Clark's avatar
Rob Clark committed
152
rebind_resource(struct fd_resource *rsc) assert_dt
153
{
Rob Clark's avatar
Rob Clark committed
154
   struct fd_screen *screen = fd_screen(rsc->b.b.screen);
155

Rob Clark's avatar
Rob Clark committed
156
157
   fd_screen_lock(screen);
   fd_resource_lock(rsc);
158

Rob Clark's avatar
Rob Clark committed
159
160
161
   if (rsc->dirty)
      list_for_each_entry (struct fd_context, ctx, &screen->context_list, node)
         rebind_resource_in_ctx(ctx, rsc);
162

Rob Clark's avatar
Rob Clark committed
163
164
   fd_resource_unlock(rsc);
   fd_screen_unlock(screen);
165
166
}

167
168
169
static inline void
fd_resource_set_bo(struct fd_resource *rsc, struct fd_bo *bo)
{
Rob Clark's avatar
Rob Clark committed
170
   struct fd_screen *screen = fd_screen(rsc->b.b.screen);
171

Rob Clark's avatar
Rob Clark committed
172
173
   rsc->bo = bo;
   rsc->seqno = p_atomic_inc_return(&screen->rsc_seqno);
174
175
}

176
int
Rob Clark's avatar
Rob Clark committed
177
178
__fd_resource_wait(struct fd_context *ctx, struct fd_resource *rsc, unsigned op,
                   const char *func)
179
{
180
   if (op & FD_BO_PREP_NOSYNC)
Rob Clark's avatar
Rob Clark committed
181
      return fd_bo_cpu_prep(rsc->bo, ctx->pipe, op);
182

Rob Clark's avatar
Rob Clark committed
183
   int ret;
184

Rob Clark's avatar
Rob Clark committed
185
186
   perf_time_ctx (ctx, 10000, "%s: a busy \"%" PRSC_FMT "\" BO stalled", func,
                  PRSC_ARGS(&rsc->b.b)) {
Rob Clark's avatar
Rob Clark committed
187
188
      ret = fd_bo_cpu_prep(rsc->bo, ctx->pipe, op);
   }
189

Rob Clark's avatar
Rob Clark committed
190
   return ret;
191
192
}

193
194
195
static void
realloc_bo(struct fd_resource *rsc, uint32_t size)
{
Rob Clark's avatar
Rob Clark committed
196
197
198
   struct pipe_resource *prsc = &rsc->b.b;
   struct fd_screen *screen = fd_screen(rsc->b.b.screen);
   uint32_t flags =
199
      COND(prsc->usage & PIPE_USAGE_STAGING, FD_BO_CACHED_COHERENT) |
200
      COND(prsc->bind & PIPE_BIND_SCANOUT, FD_BO_SCANOUT);
Rob Clark's avatar
Rob Clark committed
201
202
   /* TODO other flags? */

203
204
205
206
207
208
209
   /* Special mode to speed up readpix/etc when there is no staging
    * transfer blit, ie. for piglit/dEQP which is more CPU limited
    * than GPU limited.
    */
   if (FD_DBG(CACHEDLINEAR) && !rsc->layout.tile_mode)
      flags |= FD_BO_CACHED_COHERENT;

Rob Clark's avatar
Rob Clark committed
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
   /* if we start using things other than write-combine,
    * be sure to check for PIPE_RESOURCE_FLAG_MAP_COHERENT
    */

   if (rsc->bo)
      fd_bo_del(rsc->bo);

   struct fd_bo *bo =
      fd_bo_new(screen->dev, size, flags, "%ux%ux%u@%u:%x", prsc->width0,
                prsc->height0, prsc->depth0, rsc->layout.cpp, prsc->bind);
   fd_resource_set_bo(rsc, bo);

   /* Zero out the UBWC area on allocation.  This fixes intermittent failures
    * with UBWC, which I suspect are due to the HW having a hard time
    * interpreting arbitrary values populating the flags buffer when the BO
    * was recycled through the bo cache (instead of fresh allocations from
    * the kernel, which are zeroed).  sleep(1) in this spot didn't work
    * around the issue, but any memset value seems to.
    */
   if (rsc->layout.ubwc) {
      rsc->needs_ubwc_clear = true;
   }

   util_range_set_empty(&rsc->valid_buffer_range);
   fd_bc_invalidate_resource(rsc, true);
235
236
}

237
static void
Rob Clark's avatar
Rob Clark committed
238
239
do_blit(struct fd_context *ctx, const struct pipe_blit_info *blit,
        bool fallback) assert_dt
240
{
Rob Clark's avatar
Rob Clark committed
241
242
   struct pipe_context *pctx = &ctx->base;

243
244
245
   assert(!ctx->in_blit);
   ctx->in_blit = true;

Rob Clark's avatar
Rob Clark committed
246
247
248
249
250
251
252
253
   /* TODO size threshold too?? */
   if (fallback || !fd_blit(pctx, blit)) {
      /* do blit on cpu: */
      util_resource_copy_region(pctx, blit->dst.resource, blit->dst.level,
                                blit->dst.box.x, blit->dst.box.y,
                                blit->dst.box.z, blit->src.resource,
                                blit->src.level, &blit->src.box);
   }
254
255

   ctx->in_blit = false;
256
257
}

258
259
260
261
262
263
/**
 * Replace the storage of dst with src.  This is only used by TC in the
 * DISCARD_WHOLE_RESOURCE path, and src is a freshly allocated buffer.
 */
void
fd_replace_buffer_storage(struct pipe_context *pctx, struct pipe_resource *pdst,
264
265
                          struct pipe_resource *psrc, unsigned num_rebinds, uint32_t rebind_mask,
                          uint32_t delete_buffer_id)
266
{
Rob Clark's avatar
Rob Clark committed
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
   struct fd_context *ctx = fd_context(pctx);
   struct fd_resource *dst = fd_resource(pdst);
   struct fd_resource *src = fd_resource(psrc);

   DBG("pdst=%p, psrc=%p", pdst, psrc);

   /* This should only be called with buffers.. which side-steps some tricker
    * cases, like a rsc that is in a batch-cache key...
    */
   assert(pdst->target == PIPE_BUFFER);
   assert(psrc->target == PIPE_BUFFER);
   assert(dst->track->bc_batch_mask == 0);
   assert(src->track->bc_batch_mask == 0);
   assert(src->track->batch_mask == 0);
   assert(src->track->write_batch == NULL);
   assert(memcmp(&dst->layout, &src->layout, sizeof(dst->layout)) == 0);

   /* get rid of any references that batch-cache might have to us (which
    * should empty/destroy rsc->batches hashset)
    *
    * Note that we aren't actually destroying dst, but we are replacing
    * it's storage so we want to go thru the same motions of decoupling
    * it's batch connections.
    */
   fd_bc_invalidate_resource(dst, true);
   rebind_resource(dst);

294
295
   util_idalloc_mt_free(&ctx->screen->buffer_ids, delete_buffer_id);

Rob Clark's avatar
Rob Clark committed
296
297
298
299
300
301
302
303
304
305
306
   fd_screen_lock(ctx->screen);

   fd_bo_del(dst->bo);
   dst->bo = fd_bo_ref(src->bo);

   fd_resource_tracking_reference(&dst->track, src->track);
   src->is_replacement = true;

   dst->seqno = p_atomic_inc_return(&ctx->screen->rsc_seqno);

   fd_screen_unlock(ctx->screen);
307
308
}

309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
static unsigned
translate_usage(unsigned usage)
{
   uint32_t op = 0;

   if (usage & PIPE_MAP_READ)
      op |= FD_BO_PREP_READ;

   if (usage & PIPE_MAP_WRITE)
      op |= FD_BO_PREP_WRITE;

   return op;
}

bool
fd_resource_busy(struct pipe_screen *pscreen, struct pipe_resource *prsc,
                 unsigned usage)
{
   struct fd_resource *rsc = fd_resource(prsc);

329
330
331
   if (pending(rsc, !!(usage & PIPE_MAP_WRITE)))
      return true;

332
333
334
335
336
337
   if (resource_busy(rsc, translate_usage(usage)))
      return true;

   return false;
}

Rob Clark's avatar
Rob Clark committed
338
339
static void flush_resource(struct fd_context *ctx, struct fd_resource *rsc,
                           unsigned usage);
340

341
342
343
344
345
346
347
348
349
350
351
352
353
354
/**
 * Helper to check if the format is something that we can blit/render
 * to.. if the format is not renderable, there is no point in trying
 * to do a staging blit (as it will still end up being a cpu copy)
 */
static bool
is_renderable(struct pipe_resource *prsc)
{
   struct pipe_screen *pscreen = prsc->screen;
   return pscreen->is_format_supported(
         pscreen, prsc->format, prsc->target, prsc->nr_samples,
         prsc->nr_storage_samples, PIPE_BIND_RENDER_TARGET);
}

355
356
357
358
359
360
/**
 * @rsc: the resource to shadow
 * @level: the level to discard (if box != NULL, otherwise ignored)
 * @box: the box to discard (or NULL if none)
 * @modifier: the modifier for the new buffer state
 */
361
362
static bool
fd_try_shadow_resource(struct fd_context *ctx, struct fd_resource *rsc,
Rob Clark's avatar
Rob Clark committed
363
364
                       unsigned level, const struct pipe_box *box,
                       uint64_t modifier) assert_dt
365
{
Rob Clark's avatar
Rob Clark committed
366
367
   struct pipe_context *pctx = &ctx->base;
   struct pipe_resource *prsc = &rsc->b.b;
368
369
   struct fd_screen *screen = fd_screen(pctx->screen);
   struct fd_batch *batch;
Rob Clark's avatar
Rob Clark committed
370
371
372
373
374
   bool fallback = false;

   if (prsc->next)
      return false;

375
376
377
378
379
380
   /* Flush any pending batches writing the resource before we go mucking around
    * in its insides.  The blit would immediately cause the batch to be flushed,
    * anyway.
    */
   fd_bc_flush_writer(ctx, rsc);

381
382
383
384
385
   /* Because IB1 ("gmem") cmdstream is built only when we flush the
    * batch, we need to flush any batches that reference this rsc as
    * a render target.  Otherwise the framebuffer state emitted in
    * IB1 will reference the resources new state, and not the state
    * at the point in time that the earlier draws referenced it.
386
387
388
    *
    * Note that being in the gmem key doesn't necessarily mean the
    * batch was considered a writer!
389
390
391
392
393
    */
   foreach_batch (batch, &screen->batch_cache, rsc->track->bc_batch_mask) {
      fd_batch_flush(batch);
   }

Rob Clark's avatar
Rob Clark committed
394
395
396
   /* TODO: somehow munge dimensions and format to copy unsupported
    * render target format to something that is supported?
    */
397
   if (!is_renderable(prsc))
Rob Clark's avatar
Rob Clark committed
398
399
      fallback = true;

400
401
402
403
404
   /* do shadowing back-blits on the cpu for buffers -- requires about a page of
    * DMA to make GPU copies worth it according to robclark.  Note, if you
    * decide to do it on the GPU then you'll need to update valid_buffer_range
    * in the swap()s below.
    */
Rob Clark's avatar
Rob Clark committed
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
   if (prsc->target == PIPE_BUFFER)
      fallback = true;

   bool discard_whole_level = box && util_texrange_covers_whole_level(
                                        prsc, level, box->x, box->y, box->z,
                                        box->width, box->height, box->depth);

   /* TODO need to be more clever about current level */
   if ((prsc->target >= PIPE_TEXTURE_2D) && box && !discard_whole_level)
      return false;

   struct pipe_resource *pshadow = pctx->screen->resource_create_with_modifiers(
      pctx->screen, prsc, &modifier, 1);

   if (!pshadow)
      return false;

   assert(!ctx->in_shadow);
   ctx->in_shadow = true;

   /* get rid of any references that batch-cache might have to us (which
    * should empty/destroy rsc->batches hashset)
    */
   fd_bc_invalidate_resource(rsc, false);
   rebind_resource(rsc);

   fd_screen_lock(ctx->screen);

   /* Swap the backing bo's, so shadow becomes the old buffer,
    * blit from shadow to new buffer.  From here on out, we
    * cannot fail.
    *
    * Note that we need to do it in this order, otherwise if
    * we go down cpu blit path, the recursive transfer_map()
    * sees the wrong status..
    */
   struct fd_resource *shadow = fd_resource(pshadow);

   DBG("shadow: %p (%d, %p) -> %p (%d, %p)", rsc, rsc->b.b.reference.count,
       rsc->track, shadow, shadow->b.b.reference.count, shadow->track);

   swap(rsc->bo, shadow->bo);
447
   swap(rsc->valid, shadow->valid);
448
449
450
451
452
453

   /* swap() doesn't work because you can't typeof() the bitfield. */
   bool temp = shadow->needs_ubwc_clear;
   shadow->needs_ubwc_clear = rsc->needs_ubwc_clear;
   rsc->needs_ubwc_clear = temp;

Rob Clark's avatar
Rob Clark committed
454
455
456
457
458
459
460
461
   swap(rsc->layout, shadow->layout);
   rsc->seqno = p_atomic_inc_return(&ctx->screen->rsc_seqno);

   /* at this point, the newly created shadow buffer is not referenced
    * by any batches, but the existing rsc (probably) is.  We need to
    * transfer those references over:
    */
   debug_assert(shadow->track->batch_mask == 0);
462
   foreach_batch (batch, &ctx->screen->batch_cache, rsc->track->batch_mask) {
Rob Clark's avatar
Rob Clark committed
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
      struct set_entry *entry = _mesa_set_search(batch->resources, rsc);
      _mesa_set_remove(batch->resources, entry);
      _mesa_set_add(batch->resources, shadow);
   }
   swap(rsc->track, shadow->track);

   fd_screen_unlock(ctx->screen);

   struct pipe_blit_info blit = {};
   blit.dst.resource = prsc;
   blit.dst.format = prsc->format;
   blit.src.resource = pshadow;
   blit.src.format = pshadow->format;
   blit.mask = util_format_get_mask(prsc->format);
   blit.filter = PIPE_TEX_FILTER_NEAREST;

#define set_box(field, val)                                                    \
   do {                                                                        \
      blit.dst.field = (val);                                                  \
      blit.src.field = (val);                                                  \
   } while (0)

   /* Disable occlusion queries during shadow blits. */
   bool saved_active_queries = ctx->active_queries;
   pctx->set_active_query_state(pctx, false);

   /* blit the other levels in their entirety: */
   for (unsigned l = 0; l <= prsc->last_level; l++) {
      if (box && l == level)
         continue;

      /* just blit whole level: */
      set_box(level, l);
      set_box(box.width, u_minify(prsc->width0, l));
      set_box(box.height, u_minify(prsc->height0, l));
      set_box(box.depth, u_minify(prsc->depth0, l));

      for (int i = 0; i < prsc->array_size; i++) {
         set_box(box.z, i);
         do_blit(ctx, &blit, fallback);
      }
   }

   /* deal w/ current level specially, since we might need to split
    * it up into a couple blits:
    */
   if (box && !discard_whole_level) {
      set_box(level, level);

      switch (prsc->target) {
      case PIPE_BUFFER:
      case PIPE_TEXTURE_1D:
         set_box(box.y, 0);
         set_box(box.z, 0);
         set_box(box.height, 1);
         set_box(box.depth, 1);

         if (box->x > 0) {
            set_box(box.x, 0);
            set_box(box.width, box->x);

            do_blit(ctx, &blit, fallback);
         }
         if ((box->x + box->width) < u_minify(prsc->width0, level)) {
            set_box(box.x, box->x + box->width);
            set_box(box.width,
                    u_minify(prsc->width0, level) - (box->x + box->width));

            do_blit(ctx, &blit, fallback);
         }
         break;
      case PIPE_TEXTURE_2D:
         /* TODO */
      default:
         unreachable("TODO");
      }
   }

   pctx->set_active_query_state(pctx, saved_active_queries);

   ctx->in_shadow = false;

   pipe_resource_reference(&pshadow, NULL);

   return true;
548
549
}

550
551
552
553
/**
 * Uncompress an UBWC compressed buffer "in place".  This works basically
 * like resource shadowing, creating a new resource, and doing an uncompress
 * blit, and swapping the state between shadow and original resource so it
554
 * appears to the gallium frontends as if nothing changed.
555
556
 */
void
557
fd_resource_uncompress(struct fd_context *ctx, struct fd_resource *rsc, bool linear)
558
{
Rob Clark's avatar
Rob Clark committed
559
   tc_assert_driver_thread(ctx->tc);
560

561
562
563
   uint64_t modifier = linear ? DRM_FORMAT_MOD_LINEAR : FD_FORMAT_MOD_QCOM_TILED;

   bool success = fd_try_shadow_resource(ctx, rsc, 0, NULL, modifier);
564

Rob Clark's avatar
Rob Clark committed
565
566
   /* shadow should not fail in any cases where we need to uncompress: */
   debug_assert(success);
567
568
}

569
570
571
572
573
574
/**
 * Debug helper to hexdump a resource.
 */
void
fd_resource_dump(struct fd_resource *rsc, const char *name)
{
575
   fd_bo_cpu_prep(rsc->bo, NULL, FD_BO_PREP_READ);
Rob Clark's avatar
Rob Clark committed
576
577
   printf("%s: \n", name);
   dump_hex(fd_bo_map(rsc->bo), fd_bo_size(rsc->bo));
578
579
}

580
581
static struct fd_resource *
fd_alloc_staging(struct fd_context *ctx, struct fd_resource *rsc,
Rob Clark's avatar
Rob Clark committed
582
                 unsigned level, const struct pipe_box *box)
583
   assert_dt
584
{
Rob Clark's avatar
Rob Clark committed
585
586
587
   struct pipe_context *pctx = &ctx->base;
   struct pipe_resource tmpl = rsc->b.b;

588
589
590
   /* We cannot currently do stencil export on earlier gens, and
    * u_blitter cannot do blits involving stencil otherwise:
    */
591
   if ((ctx->screen->gen < 6) && !ctx->blit &&
592
593
594
       (util_format_get_mask(tmpl.format) & PIPE_MASK_S))
      return NULL;

Rob Clark's avatar
Rob Clark committed
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
   tmpl.width0 = box->width;
   tmpl.height0 = box->height;
   /* for array textures, box->depth is the array_size, otherwise
    * for 3d textures, it is the depth:
    */
   if (tmpl.array_size > 1) {
      if (tmpl.target == PIPE_TEXTURE_CUBE)
         tmpl.target = PIPE_TEXTURE_2D_ARRAY;
      tmpl.array_size = box->depth;
      tmpl.depth0 = 1;
   } else {
      tmpl.array_size = 1;
      tmpl.depth0 = box->depth;
   }
   tmpl.last_level = 0;
   tmpl.bind |= PIPE_BIND_LINEAR;
   tmpl.usage = PIPE_USAGE_STAGING;

   struct pipe_resource *pstaging =
      pctx->screen->resource_create(pctx->screen, &tmpl);
   if (!pstaging)
      return NULL;

   return fd_resource(pstaging);
619
620
621
}

static void
Rob Clark's avatar
Rob Clark committed
622
623
fd_blit_from_staging(struct fd_context *ctx,
                     struct fd_transfer *trans) assert_dt
624
{
625
   DBG("");
Rob Clark's avatar
Rob Clark committed
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
   struct pipe_resource *dst = trans->b.b.resource;
   struct pipe_blit_info blit = {};

   blit.dst.resource = dst;
   blit.dst.format = dst->format;
   blit.dst.level = trans->b.b.level;
   blit.dst.box = trans->b.b.box;
   blit.src.resource = trans->staging_prsc;
   blit.src.format = trans->staging_prsc->format;
   blit.src.level = 0;
   blit.src.box = trans->staging_box;
   blit.mask = util_format_get_mask(trans->staging_prsc->format);
   blit.filter = PIPE_TEX_FILTER_NEAREST;

   do_blit(ctx, &blit, false);
Rob Clark's avatar
Rob Clark committed
641
642
643
}

static void
Rob Clark's avatar
Rob Clark committed
644
fd_blit_to_staging(struct fd_context *ctx, struct fd_transfer *trans) assert_dt
Rob Clark's avatar
Rob Clark committed
645
{
646
   DBG("");
Rob Clark's avatar
Rob Clark committed
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
   struct pipe_resource *src = trans->b.b.resource;
   struct pipe_blit_info blit = {};

   blit.src.resource = src;
   blit.src.format = src->format;
   blit.src.level = trans->b.b.level;
   blit.src.box = trans->b.b.box;
   blit.dst.resource = trans->staging_prsc;
   blit.dst.format = trans->staging_prsc->format;
   blit.dst.level = 0;
   blit.dst.box = trans->staging_box;
   blit.mask = util_format_get_mask(trans->staging_prsc->format);
   blit.filter = PIPE_TEX_FILTER_NEAREST;

   do_blit(ctx, &blit, false);
662
663
}

Rob Clark's avatar
Rob Clark committed
664
665
666
667
static void
fd_resource_transfer_flush_region(struct pipe_context *pctx,
                                  struct pipe_transfer *ptrans,
                                  const struct pipe_box *box)
Rob Clark's avatar
Rob Clark committed
668
{
Rob Clark's avatar
Rob Clark committed
669
   struct fd_resource *rsc = fd_resource(ptrans->resource);
670

Rob Clark's avatar
Rob Clark committed
671
672
673
674
   if (ptrans->resource->target == PIPE_BUFFER)
      util_range_add(&rsc->b.b, &rsc->valid_buffer_range,
                     ptrans->box.x + box->x,
                     ptrans->box.x + box->x + box->width);
Rob Clark's avatar
Rob Clark committed
675
676
}

677
static void
Rob Clark's avatar
Rob Clark committed
678
679
flush_resource(struct fd_context *ctx, struct fd_resource *rsc,
               unsigned usage) assert_dt
680
{
Rob Clark's avatar
Rob Clark committed
681
   if (usage & PIPE_MAP_WRITE) {
682
      fd_bc_flush_readers(ctx, rsc);
683
   } else {
684
      fd_bc_flush_writer(ctx, rsc);
Rob Clark's avatar
Rob Clark committed
685
   }
686
687
688
}

static void
Rob Clark's avatar
Rob Clark committed
689
690
fd_flush_resource(struct pipe_context *pctx, struct pipe_resource *prsc)
   in_dt
691
{
Rob Clark's avatar
Rob Clark committed
692
693
694
695
696
697
698
699
700
   struct fd_context *ctx = fd_context(pctx);
   struct fd_resource *rsc = fd_resource(prsc);

   flush_resource(ctx, rsc, PIPE_MAP_READ);

   /* If we had to flush a batch, make sure it makes it's way all the
    * way to the kernel:
    */
   fd_resource_wait(ctx, rsc, FD_BO_PREP_FLUSH);
701
702
}

Rob Clark's avatar
Rob Clark committed
703
704
static void
fd_resource_transfer_unmap(struct pipe_context *pctx,
Rob Clark's avatar
Rob Clark committed
705
706
                           struct pipe_transfer *ptrans)
   in_dt /* TODO for threaded-ctx we'll need to split out unsynchronized path */
Rob Clark's avatar
Rob Clark committed
707
{
Rob Clark's avatar
Rob Clark committed
708
709
710
   struct fd_context *ctx = fd_context(pctx);
   struct fd_resource *rsc = fd_resource(ptrans->resource);
   struct fd_transfer *trans = fd_transfer(ptrans);
711

Rob Clark's avatar
Rob Clark committed
712
713
714
715
716
   if (trans->staging_prsc) {
      if (ptrans->usage & PIPE_MAP_WRITE)
         fd_blit_from_staging(ctx, trans);
      pipe_resource_reference(&trans->staging_prsc, NULL);
   }
717

Rob Clark's avatar
Rob Clark committed
718
719
720
   if (!(ptrans->usage & PIPE_MAP_UNSYNCHRONIZED)) {
      fd_bo_cpu_fini(rsc->bo);
   }
721

Rob Clark's avatar
Rob Clark committed
722
723
   util_range_add(&rsc->b.b, &rsc->valid_buffer_range, ptrans->box.x,
                  ptrans->box.x + ptrans->box.width);
724

Rob Clark's avatar
Rob Clark committed
725
   pipe_resource_reference(&ptrans->resource, NULL);
726

Rob Clark's avatar
Rob Clark committed
727
   assert(trans->b.staging == NULL); /* for threaded context only */
728

Rob Clark's avatar
Rob Clark committed
729
730
731
732
   /* Don't use pool_transfers_unsync. We are always in the driver
    * thread. Freeing an object into a different pool is allowed.
    */
   slab_free(&ctx->transfer_pool, ptrans);
Rob Clark's avatar
Rob Clark committed
733
734
}

735
static void
Rob Clark's avatar
Rob Clark committed
736
invalidate_resource(struct fd_resource *rsc, unsigned usage) assert_dt
737
{
Rob Clark's avatar
Rob Clark committed
738
739
740
   bool needs_flush = pending(rsc, !!(usage & PIPE_MAP_WRITE));
   unsigned op = translate_usage(usage);

741
   if (needs_flush || resource_busy(rsc, op)) {
Rob Clark's avatar
Rob Clark committed
742
743
744
745
746
      rebind_resource(rsc);
      realloc_bo(rsc, fd_bo_size(rsc->bo));
   } else {
      util_range_set_empty(&rsc->valid_buffer_range);
   }
747
748
}

749
static void *
750
resource_transfer_map_unsync(struct pipe_context *pctx,
Rob Clark's avatar
Rob Clark committed
751
752
753
                             struct pipe_resource *prsc, unsigned level,
                             unsigned usage, const struct pipe_box *box,
                             struct fd_transfer *trans)
754
{
Rob Clark's avatar
Rob Clark committed
755
756
757
758
   struct fd_resource *rsc = fd_resource(prsc);
   enum pipe_format format = prsc->format;
   uint32_t offset;
   char *buf;
759

Rob Clark's avatar
Rob Clark committed
760
761
762
763
   buf = fd_bo_map(rsc->bo);
   offset = box->y / util_format_get_blockheight(format) * trans->b.b.stride +
            box->x / util_format_get_blockwidth(format) * rsc->layout.cpp +
            fd_resource_offset(rsc, level, box->z);
764

Rob Clark's avatar
Rob Clark committed
765
766
   if (usage & PIPE_MAP_WRITE)
      rsc->valid = true;
767

Rob Clark's avatar
Rob Clark committed
768
   return buf + offset;
769
}
Rob Clark's avatar
Rob Clark committed
770

771
772
773
774
775
776
/**
 * Note, with threaded_context, resource_transfer_map() is only called
 * in driver thread, but resource_transfer_map_unsync() can be called in
 * either driver or frontend thread.
 */
static void *
Rob Clark's avatar
Rob Clark committed
777
778
779
780
resource_transfer_map(struct pipe_context *pctx, struct pipe_resource *prsc,
                      unsigned level, unsigned usage,
                      const struct pipe_box *box,
                      struct fd_transfer *trans) in_dt
781
{
Rob Clark's avatar
Rob Clark committed
782
783
784
785
786
787
788
   struct fd_context *ctx = fd_context(pctx);
   struct fd_resource *rsc = fd_resource(prsc);
   char *buf;
   int ret = 0;

   tc_assert_driver_thread(ctx->tc);

789
790
791
792
793
794
795
   /* Strip the read flag if the buffer has been invalidated (or is freshly
    * created). Avoids extra staging blits of undefined data on glTexSubImage of
    * a fresh DEPTH_COMPONENT or STENCIL_INDEX texture being stored as z24s8.
    */
   if (!rsc->valid)
      usage &= ~PIPE_MAP_READ;

Rob Clark's avatar
Rob Clark committed
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
   /* we always need a staging texture for tiled buffers:
    *
    * TODO we might sometimes want to *also* shadow the resource to avoid
    * splitting a batch.. for ex, mid-frame texture uploads to a tiled
    * texture.
    */
   if (rsc->layout.tile_mode) {
      struct fd_resource *staging_rsc;

      assert(prsc->target != PIPE_BUFFER);

      staging_rsc = fd_alloc_staging(ctx, rsc, level, box);
      if (staging_rsc) {
         trans->staging_prsc = &staging_rsc->b.b;
         trans->b.b.stride = fd_resource_pitch(staging_rsc, 0);
         trans->b.b.layer_stride = fd_resource_layer_stride(staging_rsc, 0);
         trans->staging_box = *box;
         trans->staging_box.x = 0;
         trans->staging_box.y = 0;
         trans->staging_box.z = 0;

         if (usage & PIPE_MAP_READ) {
            fd_blit_to_staging(ctx, trans);

820
            fd_resource_wait(ctx, staging_rsc, FD_BO_PREP_READ);
Rob Clark's avatar
Rob Clark committed
821
822
823
824
825
826
827
828
         }

         buf = fd_bo_map(staging_rsc->bo);

         ctx->stats.staging_uploads++;

         return buf;
      }
829
830
831
   } else if (usage & PIPE_MAP_READ) {
      perf_debug_ctx(ctx, "wc readback: prsc=%p, level=%u, usage=%x, box=%dx%d+%d,%d",
                     prsc, level, usage, box->width, box->height, box->x, box->y);
Rob Clark's avatar
Rob Clark committed
832
833
834
835
836
837
838
839
840
841
842
   }

   if (usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE) {
      invalidate_resource(rsc, usage);
   } else {
      unsigned op = translate_usage(usage);
      bool needs_flush = pending(rsc, !!(usage & PIPE_MAP_WRITE));

      /* If the GPU is writing to the resource, or if it is reading from the
       * resource and we're trying to write to it, flush the renders.
       */
843
      bool busy = needs_flush || resource_busy(rsc, op);
Rob Clark's avatar
Rob Clark committed
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862

      /* if we need to flush/stall, see if we can make a shadow buffer
       * to avoid this:
       *
       * TODO we could go down this path !reorder && !busy_for_read
       * ie. we only *don't* want to go down this path if the blit
       * will trigger a flush!
       */
      if (ctx->screen->reorder && busy && !(usage & PIPE_MAP_READ) &&
          (usage & PIPE_MAP_DISCARD_RANGE)) {

         /* try shadowing only if it avoids a flush, otherwise staging would
          * be better:
          */
         if (needs_flush && fd_try_shadow_resource(ctx, rsc, level, box,
                                                   DRM_FORMAT_MOD_LINEAR)) {
            needs_flush = busy = false;
            ctx->stats.shadow_uploads++;
         } else {
863
            struct fd_resource *staging_rsc = NULL;
Rob Clark's avatar
Rob Clark committed
864
865
866
867
868
869
870
871
872
873
874

            if (needs_flush) {
               flush_resource(ctx, rsc, usage);
               needs_flush = false;
            }

            /* in this case, we don't need to shadow the whole resource,
             * since any draw that references the previous contents has
             * already had rendering flushed for all tiles.  So we can
             * use a staging buffer to do the upload.
             */
875
876
            if (is_renderable(prsc))
               staging_rsc = fd_alloc_staging(ctx, rsc, level, box);
Rob Clark's avatar
Rob Clark committed
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
            if (staging_rsc) {
               trans->staging_prsc = &staging_rsc->b.b;
               trans->b.b.stride = fd_resource_pitch(staging_rsc, 0);
               trans->b.b.layer_stride =
                  fd_resource_layer_stride(staging_rsc, 0);
               trans->staging_box = *box;
               trans->staging_box.x = 0;
               trans->staging_box.y = 0;
               trans->staging_box.z = 0;
               buf = fd_bo_map(staging_rsc->bo);

               ctx->stats.staging_uploads++;

               return buf;
            }
         }
      }

      if (needs_flush) {
         flush_resource(ctx, rsc, usage);
         needs_flush = false;
      }

      /* The GPU keeps track of how the various bo's are being used, and
       * will wait if necessary for the proper operation to have
       * completed.
       */
      if (busy) {
         ret = fd_resource_wait(ctx, rsc, op);
         if (ret)
            return NULL;
      }
   }

   return resource_transfer_map_unsync(pctx, prsc, level, usage, box, trans);
912
}
Rob Clark's avatar
Rob Clark committed
913

914
915
static unsigned
improve_transfer_map_usage(struct fd_context *ctx, struct fd_resource *rsc,
Rob Clark's avatar
Rob Clark committed
916
917
918
919
920
                           unsigned usage, const struct pipe_box *box)
   /* Not *strictly* true, but the access to things that must only be in driver-
    * thread are protected by !(usage & TC_TRANSFER_MAP_THREADED_UNSYNC):
    */
   in_dt
921
{
Rob Clark's avatar
Rob Clark committed
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
   if (usage & TC_TRANSFER_MAP_NO_INVALIDATE) {
      usage &= ~PIPE_MAP_DISCARD_WHOLE_RESOURCE;
   }

   if (usage & TC_TRANSFER_MAP_THREADED_UNSYNC)
      usage |= PIPE_MAP_UNSYNCHRONIZED;

   if (!(usage &
         (TC_TRANSFER_MAP_NO_INFER_UNSYNCHRONIZED | PIPE_MAP_UNSYNCHRONIZED))) {
      if (ctx->in_shadow && !(usage & PIPE_MAP_READ)) {
         usage |= PIPE_MAP_UNSYNCHRONIZED;
      } else if ((usage & PIPE_MAP_WRITE) && (rsc->b.b.target == PIPE_BUFFER) &&
                 !util_ranges_intersect(&rsc->valid_buffer_range, box->x,
                                        box->x + box->width)) {
         /* We are trying to write to a previously uninitialized range. No need
          * to synchronize.
          */
         usage |= PIPE_MAP_UNSYNCHRONIZED;
      }
   }

   return usage;
944
945
}

946
static void *
Rob Clark's avatar
Rob Clark committed
947
948
949
950
fd_resource_transfer_map(struct pipe_context *pctx, struct pipe_resource *prsc,
                         unsigned level, unsigned usage,
                         const struct pipe_box *box,
                         struct pipe_transfer **pptrans)
951
{
Rob Clark's avatar
Rob Clark committed
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
   struct fd_context *ctx = fd_context(pctx);
   struct fd_resource *rsc = fd_resource(prsc);
   struct fd_transfer *trans;
   struct pipe_transfer *ptrans;

   DBG("prsc=%p, level=%u, usage=%x, box=%dx%d+%d,%d", prsc, level, usage,
       box->width, box->height, box->x, box->y);

   if ((usage & PIPE_MAP_DIRECTLY) && rsc->layout.tile_mode) {
      DBG("CANNOT MAP DIRECTLY!\n");
      return NULL;
   }

   if (usage & TC_TRANSFER_MAP_THREADED_UNSYNC) {
      ptrans = slab_alloc(&ctx->transfer_pool_unsync);
   } else {
      ptrans = slab_alloc(&ctx->transfer_pool);
   }

   if (!ptrans)
      return NULL;

   /* slab_alloc_st() doesn't zero: */
   trans = fd_transfer(ptrans);
   memset(trans, 0, sizeof(*trans));

   usage = improve_transfer_map_usage(ctx, rsc, usage, box);

   pipe_resource_reference(&ptrans->resource, prsc);
   ptrans->level = level;
   ptrans->usage = usage;
   ptrans->box = *box;
   ptrans->stride = fd_resource_pitch(rsc, level);
   ptrans->layer_stride = fd_resource_layer_stride(rsc, level);

   void *ret;
   if (usage & PIPE_MAP_UNSYNCHRONIZED) {
      ret = resource_transfer_map_unsync(pctx, prsc, level, usage, box, trans);
   } else {
      ret = resource_transfer_map(pctx, prsc, level, usage, box, trans);
   }

   if (ret) {
      *pptrans = ptrans;
   } else {
      fd_resource_transfer_unmap(pctx, ptrans);
   }

   return ret;
1001
1002
1003
}

static void
Rob Clark's avatar
Rob Clark committed
1004
fd_resource_destroy(struct pipe_screen *pscreen, struct pipe_resource *prsc)
1005
{
1006
   struct fd_screen *screen = fd_screen(prsc->screen);
Rob Clark's avatar
Rob Clark committed
1007
   struct fd_resource *rsc = fd_resource(prsc);
1008

Rob Clark's avatar
Rob Clark committed
1009
1010
1011
1012
1013
1014
1015
1016
   if (!rsc->is_replacement)
      fd_bc_invalidate_resource(rsc, true);
   if (rsc->bo)
      fd_bo_del(rsc->bo);
   if (rsc->lrz)
      fd_bo_del(rsc->lrz);
   if (rsc->scanout)
      renderonly_scanout_destroy(rsc->scanout, fd_screen(pscreen)->ro);
1017

1018
1019
1020
   if (prsc->target == PIPE_BUFFER)
      util_idalloc_mt_free(&screen->buffer_ids, rsc->b.buffer_id_unique);

Rob Clark's avatar
Rob Clark committed
1021
   threaded_resource_deinit(prsc);
1022

Rob Clark's avatar
Rob Clark committed
1023
1024
1025
   util_range_destroy(&rsc->valid_buffer_range);
   simple_mtx_destroy(&rsc->lock);
   fd_resource_tracking_reference(&rsc->track, NULL);
1026

Rob Clark's avatar
Rob Clark committed
1027
   FREE(rsc);
1028
1029
}

1030
1031
1032
static uint64_t
fd_resource_modifier(struct fd_resource *rsc)
{
Rob Clark's avatar
Rob Clark committed
1033
1034
   if (!rsc->layout.tile_mode)
      return DRM_FORMAT_MOD_LINEAR;
1035

Rob Clark's avatar
Rob Clark committed
1036
1037
   if (rsc->layout.ubwc_layer_size)
      return DRM_FORMAT_MOD_QCOM_COMPRESSED;
1038

Rob Clark's avatar
Rob Clark committed
1039
1040
   /* TODO invent a modifier for tiled but not UBWC buffers: */
   return DRM_FORMAT_MOD_INVALID;
1041
1042
}

1043
static bool
Rob Clark's avatar
Rob Clark committed
1044
1045
1046
fd_resource_get_handle(struct pipe_screen *pscreen, struct pipe_context *pctx,
                       struct pipe_resource *prsc, struct winsys_handle *handle,
                       unsigned usage)
1047
{
Rob Clark's avatar
Rob Clark committed
1048
   struct fd_resource *rsc = fd_resource(prsc);
1049

Rob Clark's avatar
Rob Clark committed
1050
   rsc->b.is_shared = true;
1051

Rob Clark's avatar
Rob Clark committed
1052
   handle->modifier = fd_resource_modifier(rsc);
1053

Rob Clark's avatar
Rob Clark committed
1054
   DBG("%" PRSC_FMT ", modifier=%" PRIx64, PRSC_ARGS(prsc), handle->modifier);
1055

Rob Clark's avatar
Rob Clark committed
1056
1057
   return fd_screen_bo_get_handle(pscreen, rsc->bo, rsc->scanout,
                                  fd_resource_pitch(rsc, 0), handle);
1058
1059
}

1060
1061
1062
1063
/* special case to resize query buf after allocated.. */
void
fd_resource_resize(struct pipe_resource *prsc, uint32_t sz)
{
Rob Clark's avatar
Rob Clark committed
1064
   struct fd_resource *rsc = fd_resource(prsc);
1065

Rob Clark's avatar
Rob Clark committed
1066
1067
1068
   debug_assert(prsc->width0 == 0);
   debug_assert(prsc->target == PIPE_BUFFER);
   debug_assert(prsc->bind == PIPE_BIND_QUERY_BUFFER);
1069

Rob Clark's avatar
Rob Clark committed
1070
1071
   prsc->width0 = sz;
   realloc_bo(rsc, fd_screen(prsc->screen)->setup_slices(rsc));
1072
1073
}

Rob Clark's avatar
Rob Clark committed
1074
1075
1076
static void
fd_resource_layout_init(struct pipe_resource *prsc)
{
Rob Clark's avatar
Rob Clark committed
1077
1078
   struct fd_resource *rsc = fd_resource(prsc);
   struct fdl_layout *layout = &rsc->layout;
Rob Clark's avatar
Rob Clark committed
1079

Rob Clark's avatar
Rob Clark committed
1080
   layout->format = prsc->format;
1081

Rob Clark's avatar
Rob Clark committed
1082
1083
1084
   layout->width0 = prsc->width0;
   layout->height0 = prsc->height0;
   layout->depth0 = prsc->depth0;
Rob Clark's avatar
Rob Clark committed
1085

Rob Clark's avatar
Rob Clark committed
1086
1087
1088
   layout->cpp = util_format_get_blocksize(prsc->format);
   layout->cpp *= fd_resource_nr_samples(prsc);
   layout->cpp_shift = ffs(layout->cpp) - 1;
Rob Clark's avatar
Rob Clark committed
1089
1090
}

1091
static struct fd_resource *
Rob Clark's avatar
Rob Clark committed
1092
1093
alloc_resource_struct(struct pipe_screen *pscreen,
                      const struct pipe_resource *tmpl)
1094
{
1095
   struct fd_screen *screen = fd_screen(pscreen);
Rob Clark's avatar
Rob Clark committed
1096
   struct fd_resource *rsc = CALLOC_STRUCT(fd_resource);
1097

Rob Clark's avatar
Rob Clark committed
1098
1099
   if (!rsc)
      return NULL;
1100

Rob Clark's avatar
Rob Clark committed
1101
1102
   struct pipe_resource *prsc = &rsc->b.b;
   *prsc = *tmpl;
1103

Rob Clark's avatar
Rob Clark committed
1104
1105
   pipe_reference_init(&prsc->reference, 1);
   prsc->screen = pscreen;
1106

Rob Clark's avatar
Rob Clark committed
1107
1108
   util_range_init(&rsc->valid_buffer_range);
   simple_mtx_init(&rsc->lock, mtx_plain);
1109

Rob Clark's avatar
Rob Clark committed
1110
1111
1112
1113
1114
   rsc->track = CALLOC_STRUCT(fd_resource_tracking);
   if (!rsc->track) {
      free(rsc);
      return NULL;
   }
1115

Rob Clark's avatar
Rob Clark committed
1116
   pipe_reference_init(&rsc->track->reference, 1);
1117

1118
1119
1120
1121
1122
   threaded_resource_init(prsc);

   if (tmpl->target == PIPE_BUFFER)
      rsc->b.buffer_id_unique = util_idalloc_mt_alloc(&screen->buffer_ids);

Rob Clark's avatar
Rob Clark committed
1123
   return rsc;
1124
1125
}

1126
/**
1127
1128
1129
1130
1131
 * Helper that allocates a resource and resolves its layout (but doesn't
 * allocate its bo).
 *
 * It returns a pipe_resource (as fd_resource_create_with_modifiers()
 * would do), and also bo's minimum required size as an output argument.
1132
1133
 */
static struct pipe_resource *
1134
fd_resource_allocate_and_resolve(struct pipe_screen *pscreen,
Rob Clark's avatar
Rob Clark committed
1135
1136
1137
                                 const struct pipe_resource *tmpl,
                                 const uint64_t *modifiers, int count,
                                 uint32_t *psize)
1138
{
Rob Clark's avatar
Rob Clark committed
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
   struct fd_screen *screen = fd_screen(pscreen);
   struct fd_resource *rsc;
   struct pipe_resource *prsc;
   enum pipe_format format = tmpl->format;
   uint32_t size;

   rsc = alloc_resource_struct(pscreen, tmpl);
   if (!rsc)
      return NULL;

   prsc = &rsc->b.b;

1151
1152
1153
1154
   /* Clover creates buffers with PIPE_FORMAT_NONE: */
   if ((prsc->target == PIPE_BUFFER) && (format == PIPE_FORMAT_NONE))
      format = prsc->format = PIPE_FORMAT_R8_UNORM;

Rob Clark's avatar
Rob Clark committed
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
   DBG("%" PRSC_FMT, PRSC_ARGS(prsc));

   if (tmpl->bind & PIPE_BIND_SHARED)
      rsc->b.is_shared = true;

   fd_resource_layout_init(prsc);

#define LINEAR (PIPE_BIND_SCANOUT | PIPE_BIND_LINEAR | PIPE_BIND_DISPLAY_TARGET)

   bool linear = drm_find_modifier(DRM_FORMAT_MOD_LINEAR, modifiers, count);
   if (linear) {
      perf_debug("%" PRSC_FMT ": linear: DRM_FORMAT_MOD_LINEAR requested!",
                 PRSC_ARGS(prsc));
   } else if (tmpl->bind & LINEAR) {
      if (tmpl->usage != PIPE_USAGE_STAGING)
         perf_debug("%" PRSC_FMT ": linear: LINEAR bind requested!",
                    PRSC_ARGS(prsc));
      linear = true;
   }

   if (FD_DBG(NOTILE))
      linear = true;

   /* Normally, for non-shared buffers, allow buffer compression if
    * not shared, otherwise only allow if QCOM_COMPRESSED modifier
    * is requested:
    *
    * TODO we should probably also limit tiled in a similar way,
    * except we don't have a format modifier for tiled.  (We probably
    * should.)
    */
   bool allow_ubwc = false;
   if (!linear) {
      allow_ubwc = drm_find_modifier(DRM_FORMAT_MOD_INVALID, modifiers, count);
      if (!allow_ubwc) {
         perf_debug("%" PRSC_FMT
                    ": not UBWC: DRM_FORMAT_MOD_INVALID not requested!",
                    PRSC_ARGS(prsc));
      }
      if (tmpl->bind & PIPE_BIND_SHARED) {
         allow_ubwc =
            drm_find_modifier(DRM_FORMAT_MOD_QCOM_COMPRESSED, modifiers, count);
         if (!allow_ubwc) {
            perf_debug("%" PRSC_FMT
                       ": not UBWC: shared and DRM_FORMAT_MOD_QCOM_COMPRESSED "
                       "not requested!",
                       PRSC_ARGS(prsc));
            linear = true;
         }
      }
   }

   allow_ubwc &= !FD_DBG(NOUBWC);

   if (screen->tile_mode && (tmpl->target != PIPE_BUFFER) && !linear) {
      rsc->layout.tile_mode = screen->tile_mode(prsc);
   }

   rsc->internal_format = format;

   rsc->layout.ubwc = rsc->layout.tile_mode && is_a6xx(screen) && allow_ubwc;

   if (prsc->target == PIPE_BUFFER) {
      assert(prsc->format == PIPE_FORMAT_R8_UNORM);
      size = prsc->width0;
      fdl_layout_buffer(&rsc->layout, size);
   } else {
      size = screen->setup_slices(rsc);
   }

   /* special case for hw-query buffer, which we need to allocate before we
    * know the size:
    */
   if (size == 0) {
      /* note, semi-intention == instead of & */
      debug_assert(prsc->bind == PIPE_BIND_QUERY_BUFFER);
      *psize = 0;
      return prsc;
   }

   /* Set the layer size if the (non-a6xx) backend hasn't done so. */
   if (rsc->layout.layer_first && !rsc->layout.layer_size) {
      rsc->layout.layer_size = align(size, 4096);
      size = rsc->layout.layer_size * prsc->array_size;
   }

   if (FD_DBG(LAYOUT))
      fdl_dump_layout(&rsc->layout);

   /* Hand out the resolved size. */
   if (psize)
      *psize = size;

   return prsc;
1249
1250
1251
1252
1253
1254
1255
}

/**
 * Create a new texture object, using the given template info.
 */
static struct pipe_resource *
fd_resource_create_with_modifiers(struct pipe_screen *pscreen,
Rob Clark's avatar
Rob Clark committed
1256
1257
                                  const struct pipe_resource *tmpl,
                                  const uint64_t *modifiers, int count)
1258
{
Rob Clark's avatar
Rob Clark committed
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
   struct fd_screen *screen = fd_screen(pscreen);
   struct fd_resource *rsc;
   struct pipe_resource *prsc;
   uint32_t size;

   /* when using kmsro, scanout buffers are allocated on the display device
    * create_with_modifiers() doesn't give us usage flags, so we have to
    * assume that all calls with modifiers are scanout-possible
    */
   if (screen->ro &&
       ((tmpl->bind & PIPE_BIND_SCANOUT) ||
        !(count == 1 && modifiers[0] == DRM_FORMAT_MOD_INVALID))) {
      struct pipe_resource scanout_templat = *tmpl;
      struct renderonly_scanout *scanout;
      struct winsys_handle handle;

      /* note: alignment is wrong for a6xx */
1276
      scanout_templat.width0 = align(tmpl->width0, screen->info->gmem_align_w);
Rob Clark's avatar
Rob Clark committed
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305

      scanout =
         renderonly_scanout_for_resource(&scanout_templat, screen->ro, &handle);
      if (!scanout)
         return NULL;

      renderonly_scanout_destroy(scanout, screen->ro);

      assert(handle.type == WINSYS_HANDLE_TYPE_FD);
      rsc = fd_resource(pscreen->resource_from_handle(
         pscreen, tmpl, &handle, PIPE_HANDLE_USAGE_FRAMEBUFFER_WRITE));
      close(handle.handle);
      if (!rsc)
         return NULL;

      return &rsc->b.b;
   }

   prsc =
      fd_resource_allocate_and_resolve(pscreen, tmpl, modifiers, count, &size);
   if (!prsc)
      return NULL;
   rsc = fd_resource(prsc);

   realloc_bo(rsc, size);
   if (!rsc->bo)
      goto fail;

   return prsc;
1306
fail:
Rob Clark's avatar
Rob Clark committed
1307
1308
   fd_resource_destroy(pscreen, prsc);
   return NULL;
1309
1310
}

1311
1312
static struct pipe_resource *
fd_resource_create(struct pipe_screen *pscreen,
Rob Clark's avatar
Rob Clark committed
1313
                   const struct pipe_resource *tmpl)
1314
{
Rob Clark's avatar
Rob Clark committed
1315
1316
   const uint64_t mod = DRM_FORMAT_MOD_INVALID;
   return fd_resource_create_with_modifiers(pscreen, tmpl, &mod, 1);
1317
1318
}

1319
1320
1321
1322
1323
1324
1325
/**
 * Create a texture from a winsys_handle. The handle is often created in
 * another process by first creating a pipe texture and then calling
 * resource_get_handle.
 */
static struct pipe_resource *
fd_resource_from_handle(struct pipe_screen *pscreen,
Rob Clark's avatar
Rob Clark committed
1326
1327
                        const struct pipe_resource *tmpl,
                        struct winsys_handle *handle, unsigned usage)
1328
{
Rob Clark's avatar
Rob Clark committed
1329
1330
   struct fd_screen *screen = fd_screen(pscreen);
   struct fd_resource *rsc = alloc_resource_struct(pscreen, tmpl);
1331

Rob Clark's avatar
Rob Clark committed
1332
1333
   if (!rsc)
      return NULL;
1334

Rob Clark's avatar
Rob Clark committed
1335
1336
   struct fdl_slice *slice = fd_resource_slice(rsc, 0);
   struct pipe_resource *prsc = &rsc->b.b;
1337

Rob Clark's avatar
Rob Clark committed
1338
   DBG("%" PRSC_FMT ", modifier=%" PRIx64, PRSC_ARGS(prsc), handle->modifier);
1339

Rob Clark's avatar
Rob Clark committed
1340
   rsc->b.is_shared = true;
1341

Rob Clark's avatar
Rob Clark committed
1342
   fd_resource_layout_init(prsc);
1343

Rob Clark's avatar
Rob Clark committed
1344
1345
1346
   struct fd_bo *bo = fd_screen_bo_from_handle(pscreen, handle);
   if (!bo)
      goto fail;
1347

Rob Clark's avatar
Rob Clark committed
1348
   fd_resource_set_bo(rsc, bo);
1349

Rob Clark's avatar
Rob Clark committed
1350
1351
1352
1353
   rsc->internal_format = tmpl->format;
   rsc->layout.pitch0 = handle->stride;
   slice->offset = handle->offset;
   slice->size0 = handle->stride * prsc->height0;
Rob Clark's avatar
Rob Clark committed
1354