ir3_shader.h 30.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
/*
 * Copyright (C) 2014 Rob Clark <robclark@freedesktop.org>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 * Authors:
 *    Rob Clark <robclark@freedesktop.org>
 */

#ifndef IR3_SHADER_H_
#define IR3_SHADER_H_

30
31
#include <stdio.h>

32
#include "c11/threads.h"
Rob Clark's avatar
Rob Clark committed
33
#include "compiler/nir/nir.h"
34
#include "compiler/shader_enums.h"
35
#include "util/bitscan.h"
Rob Clark's avatar
Rob Clark committed
36
#include "util/disk_cache.h"
37

38
#include "ir3_compiler.h"
39

40
41
struct glsl_type;

42
43
/* driver param indices: */
enum ir3_driver_param {
44
45
46
47
   /* compute shader driver params: */
   IR3_DP_NUM_WORK_GROUPS_X = 0,
   IR3_DP_NUM_WORK_GROUPS_Y = 1,
   IR3_DP_NUM_WORK_GROUPS_Z = 2,
48
   IR3_DP_WORK_DIM          = 3,
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
   IR3_DP_BASE_GROUP_X = 4,
   IR3_DP_BASE_GROUP_Y = 5,
   IR3_DP_BASE_GROUP_Z = 6,
   IR3_DP_SUBGROUP_SIZE = 7,
   IR3_DP_LOCAL_GROUP_SIZE_X = 8,
   IR3_DP_LOCAL_GROUP_SIZE_Y = 9,
   IR3_DP_LOCAL_GROUP_SIZE_Z = 10,
   IR3_DP_SUBGROUP_ID_SHIFT = 11,
   /* NOTE: gl_NumWorkGroups should be vec4 aligned because
    * glDispatchComputeIndirect() needs to load these from
    * the info->indirect buffer.  Keep that in mind when/if
    * adding any addition CS driver params.
    */
   IR3_DP_CS_COUNT = 12, /* must be aligned to vec4 */

   /* vertex shader driver params: */
   IR3_DP_DRAWID = 0,
   IR3_DP_VTXID_BASE = 1,
   IR3_DP_INSTID_BASE = 2,
   IR3_DP_VTXCNT_MAX = 3,
   /* user-clip-plane components, up to 8x vec4's: */
   IR3_DP_UCP0_X = 4,
   /* .... */
   IR3_DP_UCP7_W = 35,
   IR3_DP_VS_COUNT = 36 /* must be aligned to vec4 */
74
75
};

76
77
78
79
80
81
#define IR3_MAX_SHADER_BUFFERS  32
#define IR3_MAX_SHADER_IMAGES   32
#define IR3_MAX_SO_BUFFERS      4
#define IR3_MAX_SO_STREAMS      4
#define IR3_MAX_SO_OUTPUTS      64
#define IR3_MAX_UBO_PUSH_RANGES 32
82

83
84
/* mirrors SYSTEM_VALUE_BARYCENTRIC_ but starting from 0 */
enum ir3_bary {
85
86
87
88
89
90
91
92
   IJ_PERSP_PIXEL,
   IJ_PERSP_SAMPLE,
   IJ_PERSP_CENTROID,
   IJ_PERSP_SIZE,
   IJ_LINEAR_PIXEL,
   IJ_LINEAR_CENTROID,
   IJ_LINEAR_SAMPLE,
   IJ_COUNT,
93
94
};

95
96
97
98
/**
 * Description of a lowered UBO.
 */
struct ir3_ubo_info {
99
100
101
   uint32_t block;         /* Which constant block */
   uint16_t bindless_base; /* For bindless, which base register is used */
   bool bindless;
102
};
103

104
105
106
107
108
109
/**
 * Description of a range of a lowered UBO access.
 *
 * Drivers should not assume that there are not multiple disjoint
 * lowered ranges of a single UBO.
 */
110
struct ir3_ubo_range {
111
112
113
   struct ir3_ubo_info ubo;
   uint32_t offset;     /* start offset to push in the const register file */
   uint32_t start, end; /* range of block that's actually used */
114
115
116
};

struct ir3_ubo_analysis_state {
117
118
119
120
121
   struct ir3_ubo_range range[IR3_MAX_UBO_PUSH_RANGES];
   uint32_t num_enabled;
   uint32_t size;
   uint32_t
      cmdstream_size; /* for per-gen backend to stash required cmdstream size */
122
123
};

124
/**
125
 * Describes the layout of shader consts.  This includes:
126
 *   + User consts + driver lowered UBO ranges
127
128
129
130
131
132
 *   + SSBO sizes
 *   + Image sizes/dimensions
 *   + Driver params (ie. IR3_DP_*)
 *   + TFBO addresses (for generations that do not have hardware streamout)
 *   + Lowered immediates
 *
133
134
135
136
 * For consts needed to pass internal values to shader which may or may not
 * be required, rather than allocating worst-case const space, we scan the
 * shader and allocate consts as-needed:
 *
137
 *   + SSBO sizes: only needed if shader has a get_ssbo_size intrinsic
138
 *     for a given SSBO
Rob Clark's avatar
Rob Clark committed
139
140
141
 *
 *   + Image dimensions: needed to calculate pixel offset, but only for
 *     images that have a image_store intrinsic
142
143
144
145
146
147
148
 *
 * Layout of constant registers, each section aligned to vec4.  Note
 * that pointer size (ubo, etc) changes depending on generation.
 *
 *    user consts
 *    UBO addresses
 *    SSBO sizes
149
 *    image dimensions
150
 *    if (vertex shader) {
151
 *        driver params (IR3_DP_VS_COUNT)
152
153
154
 *        if (stream_output.num_outputs > 0)
 *           stream-out addresses
 *    } else if (compute_shader) {
155
156
 *        kernel params
 *        driver params (IR3_DP_CS_COUNT)
157
158
159
160
161
162
163
 *    }
 *    immediates
 *
 * Immediates go last mostly because they are inserted in the CP pass
 * after the nir -> ir3 frontend.
 *
 * Note UBO size in bytes should be aligned to vec4
164
 */
165
struct ir3_const_state {
166
167
168
169
170
171
172
173
174
175
   unsigned num_ubos;
   unsigned num_driver_params; /* scalar */

   /* UBO that should be mapped to the NIR shader's constant_data (or -1). */
   int32_t constant_data_ubo;

   struct {
      /* user const start at zero */
      unsigned ubo;
      unsigned image_dims;
176
      unsigned kernel_params;
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
      unsigned driver_param;
      unsigned tfbo;
      unsigned primitive_param;
      unsigned primitive_map;
      unsigned immediate;
   } offsets;

   struct {
      uint32_t mask;  /* bitmask of images that have image_store */
      uint32_t count; /* number of consts allocated */
      /* three const allocated per image which has image_store:
       *  + cpp         (bytes per pixel)
       *  + pitch       (y pitch)
       *  + array_pitch (z pitch)
       */
      uint32_t off[IR3_MAX_SHADER_IMAGES];
   } image_dims;

   unsigned immediates_count;
   unsigned immediates_size;
   uint32_t *immediates;

   /* State of ubo access lowered to push consts: */
   struct ir3_ubo_analysis_state ubo_state;
201
202
};

203
204
205
206
/**
 * A single output for vertex transform feedback.
 */
struct ir3_stream_output {
207
208
209
210
211
212
   unsigned register_index  : 6;  /**< 0 to 63 (OUT index) */
   unsigned start_component : 2;  /** 0 to 3 */
   unsigned num_components  : 3;  /** 1 to 4 */
   unsigned output_buffer   : 3;  /**< 0 to PIPE_MAX_SO_BUFFERS */
   unsigned dst_offset      : 16; /**< offset into the buffer in dwords */
   unsigned stream          : 2;  /**< 0 to 3 */
213
214
215
216
217
218
};

/**
 * Stream output for vertex transform feedback.
 */
struct ir3_stream_output_info {
219
220
221
222
223
224
225
226
227
228
229
230
231
   unsigned num_outputs;
   /** stride for an entire vertex for each buffer in dwords */
   uint16_t stride[IR3_MAX_SO_BUFFERS];

   /* These correspond to the VPC_SO_STREAM_CNTL fields */
   uint8_t streams_written;
   uint8_t buffer_to_stream[IR3_MAX_SO_BUFFERS];

   /**
    * Array of stream outputs, in the order they are to be written in.
    * Selected components are tightly packed into the output buffer.
    */
   struct ir3_stream_output output[IR3_MAX_SO_OUTPUTS];
232
233
};

234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
/**
 * Starting from a4xx, HW supports pre-dispatching texture sampling
 * instructions prior to scheduling a shader stage, when the
 * coordinate maps exactly to an output of the previous stage.
 */

/**
 * There is a limit in the number of pre-dispatches allowed for any
 * given stage.
 */
#define IR3_MAX_SAMPLER_PREFETCH 4

/**
 * This is the output stream value for 'cmd', as used by blob. It may
 * encode the return type (in 3 bits) but it hasn't been verified yet.
 */
250
#define IR3_SAMPLER_PREFETCH_CMD          0x4
251
#define IR3_SAMPLER_BINDLESS_PREFETCH_CMD 0x6
252
253
254
255
256

/**
 * Stream output for texture sampling pre-dispatches.
 */
struct ir3_sampler_prefetch {
257
258
259
260
261
262
263
264
265
   uint8_t src;
   uint8_t samp_id;
   uint8_t tex_id;
   uint16_t samp_bindless_id;
   uint16_t tex_bindless_id;
   uint8_t dst;
   uint8_t wrmask;
   uint8_t half_precision;
   uint8_t cmd;
266
267
};

268
269
270
/* Configuration key used to identify a shader variant.. different
 * shader variants can be used to implement features not supported
 * in hw (two sided color), binning-pass vertex shader, etc.
271
272
273
 *
 * When adding to this struct, please update ir3_shader_variant()'s debug
 * output.
274
275
 */
struct ir3_shader_key {
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
   union {
      struct {
         /*
          * Combined Vertex/Fragment shader parameters:
          */
         unsigned ucp_enables : 8;

         /* do we need to check {v,f}saturate_{s,t,r}? */
         unsigned has_per_samp : 1;

         /*
          * Fragment shader variant parameters:
          */
         unsigned sample_shading : 1;
         unsigned msaa           : 1;
         /* used when shader needs to handle flat varyings (a4xx)
          * for front/back color inputs to frag shader:
          */
         unsigned rasterflat : 1;

         /* Indicates that this is a tessellation pipeline which requires a
          * whole different kind of vertex shader.  In case of
          * tessellation, this field also tells us which kind of output
          * topology the TES uses, which the TCS needs to know.
          */
#define IR3_TESS_NONE      0
#define IR3_TESS_TRIANGLES 1
#define IR3_TESS_QUADS     2
#define IR3_TESS_ISOLINES  3
         unsigned tessellation : 2;

         unsigned has_gs : 1;

309
310
311
312
313
         /* Whether stages after TCS read gl_PrimitiveID, used to determine
          * whether the TCS has to store it in the tess factor BO.
          */
         unsigned tcs_store_primid : 1;

314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
         /* Whether this variant sticks to the "safe" maximum constlen,
          * which guarantees that the combined stages will never go over
          * the limit:
          */
         unsigned safe_constlen : 1;

         /* Whether gl_Layer must be forced to 0 because it isn't written. */
         unsigned layer_zero : 1;

         /* Whether gl_ViewportIndex must be forced to 0 because it isn't
          * written. */
         unsigned view_zero : 1;
      };
      uint32_t global;
   };

   /* bitmask of ms shifts (a3xx) */
   uint32_t vsamples, fsamples;

   /* bitmask of samplers which need astc srgb workaround (a4xx+a5xx): */
   uint16_t vastc_srgb, fastc_srgb;
335
336
};

337
338
339
static inline unsigned
ir3_tess_mode(unsigned gl_tess_mode)
{
340
341
342
343
344
345
346
347
348
349
   switch (gl_tess_mode) {
   case GL_ISOLINES:
      return IR3_TESS_ISOLINES;
   case GL_TRIANGLES:
      return IR3_TESS_TRIANGLES;
   case GL_QUADS:
      return IR3_TESS_QUADS;
   default:
      unreachable("bad tessmode");
   }
350
351
}

352
static inline bool
353
354
ir3_shader_key_equal(const struct ir3_shader_key *a,
                     const struct ir3_shader_key *b)
355
{
356
357
358
359
   /* slow-path if we need to check {v,f}saturate_{s,t,r} */
   if (a->has_per_samp || b->has_per_samp)
      return memcmp(a, b, sizeof(struct ir3_shader_key)) == 0;
   return a->global == b->global;
360
361
}

362
363
/* will the two keys produce different lowering for a fragment shader? */
static inline bool
364
365
ir3_shader_key_changes_fs(struct ir3_shader_key *key,
                          struct ir3_shader_key *last_key)
366
{
367
368
369
370
371
   if (last_key->has_per_samp || key->has_per_samp) {
      if ((last_key->fsamples != key->fsamples) ||
          (last_key->fastc_srgb != key->fastc_srgb))
         return true;
   }
372

373
374
   if (last_key->rasterflat != key->rasterflat)
      return true;
375

376
377
   if (last_key->layer_zero != key->layer_zero)
      return true;
378

379
380
   if (last_key->ucp_enables != key->ucp_enables)
      return true;
381

382
383
   if (last_key->safe_constlen != key->safe_constlen)
      return true;
384

385
   return false;
386
387
388
389
}

/* will the two keys produce different lowering for a vertex shader? */
static inline bool
390
391
ir3_shader_key_changes_vs(struct ir3_shader_key *key,
                          struct ir3_shader_key *last_key)
392
{
393
394
395
396
397
   if (last_key->has_per_samp || key->has_per_samp) {
      if ((last_key->vsamples != key->vsamples) ||
          (last_key->vastc_srgb != key->vastc_srgb))
         return true;
   }
398

399
400
   if (last_key->ucp_enables != key->ucp_enables)
      return true;
401

402
403
   if (last_key->safe_constlen != key->safe_constlen)
      return true;
404

405
   return false;
406
407
}

408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
/**
 * On a4xx+a5xx, Images share state with textures and SSBOs:
 *
 *   + Uses texture (cat5) state/instruction (isam) to read
 *   + Uses SSBO state and instructions (cat6) to write and for atomics
 *
 * Starting with a6xx, Images and SSBOs are basically the same thing,
 * with texture state and isam also used for SSBO reads.
 *
 * On top of that, gallium makes the SSBO (shader_buffers) state semi
 * sparse, with the first half of the state space used for atomic
 * counters lowered to atomic buffers.  We could ignore this, but I
 * don't think we could *really* handle the case of a single shader
 * that used the max # of textures + images + SSBOs.  And once we are
 * offsetting images by num_ssbos (or visa versa) to map them into
 * the same hardware state, the hardware state has become coupled to
 * the shader state, so at this point we might as well just use a
 * mapping table to remap things from image/SSBO idx to hw idx.
 *
 * To make things less (more?) confusing, for the hw "SSBO" state
 * (since it is really both SSBO and Image) I'll use the name "IBO"
 */
struct ir3_ibo_mapping {
#define IBO_INVALID 0xff
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
   /* Maps logical SSBO state to hw tex state: */
   uint8_t ssbo_to_tex[IR3_MAX_SHADER_BUFFERS];

   /* Maps logical Image state to hw tex state: */
   uint8_t image_to_tex[IR3_MAX_SHADER_IMAGES];

   /* Maps hw state back to logical SSBO or Image state:
    *
    * note IBO_SSBO ORd into values to indicate that the
    * hw slot is used for SSBO state vs Image state.
    */
#define IBO_SSBO 0x80
   uint8_t tex_to_image[32];

   /* including real textures */
   uint8_t num_tex;
   /* the number of real textures, ie. image/ssbo start here */
   uint8_t tex_base;
450
451
};

452
struct ir3_disasm_info {
453
454
455
   bool write_disasm;
   char *nir;
   char *disasm;
456
457
};

458
/* Represents half register in regid */
459
#define HALF_REG_ID 0x100
460

461
462
463
464
/**
 * Shader variant which contains the actual hw shader instructions,
 * and necessary info for shader state setup.
 */
465
struct ir3_shader_variant {
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
   struct fd_bo *bo;

   /* variant id (for debug) */
   uint32_t id;

   struct ir3_shader_key key;

   /* vertex shaders can have an extra version for hwbinning pass,
    * which is pointed to by so->binning:
    */
   bool binning_pass;
   //	union {
   struct ir3_shader_variant *binning;
   struct ir3_shader_variant *nonbinning;
   //	};

   struct ir3 *ir; /* freed after assembling machine instructions */

   /* shader variants form a linked list: */
   struct ir3_shader_variant *next;

   /* replicated here to avoid passing extra ptrs everywhere: */
   gl_shader_stage type;
   struct ir3_shader *shader;

   /* variant's copy of nir->constant_data (since we don't track the NIR in
    * the variant, and shader->nir is before the opt pass).  Moves to v->bin
    * after assembly.
    */
   void *constant_data;

   /*
    * Below here is serialized when written to disk cache:
    */

   /* The actual binary shader instructions, size given by info.sizedwords: */
   uint32_t *bin;

   struct ir3_const_state *const_state;

   /*
    * The following macros are used by the shader disk cache save/
    * restore paths to serialize/deserialize the variant.  Any
    * pointers that require special handling in store_variant()
    * and retrieve_variant() should go above here.
    */
#define VARIANT_CACHE_START  offsetof(struct ir3_shader_variant, info)
#define VARIANT_CACHE_PTR(v) (((char *)v) + VARIANT_CACHE_START)
#define VARIANT_CACHE_SIZE                                                     \
   (sizeof(struct ir3_shader_variant) - VARIANT_CACHE_START)

   struct ir3_info info;

   uint32_t constant_data_size;

   /* Levels of nesting of flow control:
    */
   unsigned branchstack;

   unsigned max_sun;
   unsigned loops;

   /* the instructions length is in units of instruction groups
    * (4 instructions for a3xx, 16 instructions for a4xx.. each
    * instruction is 2 dwords):
    */
   unsigned instrlen;

   /* the constants length is in units of vec4's, and is the sum of
    * the uniforms and the built-in compiler constants
    */
   unsigned constlen;

   /* The private memory size in bytes */
   unsigned pvtmem_size;
   /* Whether we should use the new per-wave layout rather than per-fiber. */
   bool pvtmem_per_wave;

   /* Size in bytes of required shared memory */
   unsigned shared_size;

   /* About Linkage:
    *   + Let the frag shader determine the position/compmask for the
    *     varyings, since it is the place where we know if the varying
    *     is actually used, and if so, which components are used.  So
    *     what the hw calls "outloc" is taken from the "inloc" of the
    *     frag shader.
    *   + From the vert shader, we only need the output regid
    */

   bool frag_face, color0_mrt;
   uint8_t fragcoord_compmask;

   /* NOTE: for input/outputs, slot is:
    *   gl_vert_attrib  - for VS inputs
    *   gl_varying_slot - for VS output / FS input
    *   gl_frag_result  - for FS output
    */

   /* varyings/outputs: */
   unsigned outputs_count;
   struct {
      uint8_t slot;
      uint8_t regid;
      uint8_t view;
      bool half : 1;
   } outputs[32 + 2]; /* +POSITION +PSIZE */
   bool writes_pos, writes_smask, writes_psize, writes_stencilref;

   /* Size in dwords of all outputs for VS, size of entire patch for HS. */
   uint32_t output_size;

   /* Expected size of incoming output_loc for HS, DS, and GS */
   uint32_t input_size;

   /* Map from location to offset in per-primitive storage. In dwords for
    * HS, where varyings are read in the next stage via ldg with a dword
    * offset, and in bytes for all other stages.
    */
   unsigned output_loc[32 + 4]; /* +POSITION +PSIZE +CLIP_DIST0 +CLIP_DIST1 */

   /* attributes (VS) / varyings (FS):
    * Note that sysval's should come *after* normal inputs.
    */
   unsigned inputs_count;
   struct {
      uint8_t slot;
      uint8_t regid;
      uint8_t compmask;
      /* location of input (ie. offset passed to bary.f, etc).  This
       * matches the SP_VS_VPC_DST_REG.OUTLOCn value (a3xx and a4xx
       * have the OUTLOCn value offset by 8, presumably to account
       * for gl_Position/gl_PointSize)
       */
      uint8_t inloc;
      /* vertex shader specific: */
      bool sysval : 1; /* slot is a gl_system_value */
      /* fragment shader specific: */
      bool bary       : 1; /* fetched varying (vs one loaded into reg) */
      bool rasterflat : 1; /* special handling for emit->rasterflat */
      bool half       : 1;
      bool flat       : 1;
   } inputs[32 + 2]; /* +POSITION +FACE */

   /* sum of input components (scalar).  For frag shaders, it only counts
    * the varying inputs:
    */
   unsigned total_in;

   /* sum of sysval input components (scalar). */
   unsigned sysval_in;

   /* For frag shaders, the total number of inputs (not scalar,
    * ie. SP_VS_PARAM_REG.TOTALVSOUTVAR)
    */
   unsigned varying_in;

   /* Remapping table to map Image and SSBO to hw state: */
   struct ir3_ibo_mapping image_mapping;

   /* number of samplers/textures (which are currently 1:1): */
   int num_samp;

   /* is there an implicit sampler to read framebuffer (FS only).. if
    * so the sampler-idx is 'num_samp - 1' (ie. it is appended after
    * the last "real" texture)
    */
   bool fb_read;

   /* do we have one or more SSBO instructions: */
   bool has_ssbo;

   /* Which bindless resources are used, for filling out sp_xs_config */
   bool bindless_tex;
   bool bindless_samp;
   bool bindless_ibo;
   bool bindless_ubo;

   /* do we need derivatives: */
   bool need_pixlod;

   bool need_fine_derivatives;

   /* do we need VS driver params? */
   bool need_driver_params;

   /* do we have image write, etc (which prevents early-z): */
   bool no_earlyz;

   /* do we have kill, which also prevents early-z, but not necessarily
    * early-lrz (as long as lrz-write is disabled, which must be handled
    * outside of ir3.  Unlike other no_earlyz cases, kill doesn't have
    * side effects that prevent early-lrz discard.
    */
   bool has_kill;

   bool per_samp;

   /* Are we using split or merged register file? */
   bool mergedregs;

   uint8_t clip_mask, cull_mask;

   /* for astc srgb workaround, the number/base of additional
    * alpha tex states we need, and index of original tex states
    */
   struct {
      unsigned base, count;
      unsigned orig_idx[16];
   } astc_srgb;

   /* texture sampler pre-dispatches */
   uint32_t num_sampler_prefetch;
   struct ir3_sampler_prefetch sampler_prefetch[IR3_MAX_SAMPLER_PREFETCH];

   uint16_t local_size[3];
   bool local_size_variable;

   struct ir3_disasm_info disasm_info;
685
686
};

Rob Clark's avatar
Rob Clark committed
687
688
689
static inline const char *
ir3_shader_stage(struct ir3_shader_variant *v)
{
690
691
692
693
694
695
696
697
698
699
700
701
   switch (v->type) {
   case MESA_SHADER_VERTEX:
      return v->binning_pass ? "BVERT" : "VERT";
   case MESA_SHADER_TESS_CTRL:
      return "TCS";
   case MESA_SHADER_TESS_EVAL:
      return "TES";
   case MESA_SHADER_GEOMETRY:
      return "GEOM";
   case MESA_SHADER_FRAGMENT:
      return "FRAG";
   case MESA_SHADER_COMPUTE:
702
   case MESA_SHADER_KERNEL:
703
704
705
706
707
      return "CL";
   default:
      unreachable("invalid type");
      return NULL;
   }
Rob Clark's avatar
Rob Clark committed
708
709
}

710
711
712
713
714
715
716
/* Currently we do not do binning for tess.  And for GS there is no
 * cross-stage VS+GS optimization, so the full VS+GS is used in
 * the binning pass.
 */
static inline bool
ir3_has_binning_vs(const struct ir3_shader_key *key)
{
717
718
719
   if (key->tessellation || key->has_gs)
      return false;
   return true;
720
}
721

722
723
724
725
/**
 * Represents a shader at the API level, before state-specific variants are
 * generated.
 */
726
struct ir3_shader {
727
   gl_shader_stage type;
728

729
730
731
   /* shader id (for debug): */
   uint32_t id;
   uint32_t variant_count;
Rob Clark's avatar
Rob Clark committed
732

733
734
735
736
   /* Set by freedreno after shader_state_create, so we can emit debug info
    * when recompiling a shader at draw time.
    */
   bool initial_variants_done;
737

738
   struct ir3_compiler *compiler;
739

740
   unsigned num_reserved_user_consts;
741

742
743
744
   bool nir_finalized;
   struct nir_shader *nir;
   struct ir3_stream_output_info stream_output;
745

746
747
748
749
750
   /* per shader stage specific info: */
   union {
      /* for compute shaders: */
      struct {
         unsigned req_input_mem;    /* in dwords */
751
         unsigned req_local_mem;
752
753
754
      } cs;
   };

755
756
   struct ir3_shader_variant *variants;
   mtx_t variants_lock;
757

758
   cache_key cache_key; /* shader disk-cache key */
Rob Clark's avatar
Rob Clark committed
759

760
761
762
763
   /* Bitmask of bits of the shader key used by this shader.  Used to avoid
    * recompiles for GL NOS that doesn't actually apply to the shader.
    */
   struct ir3_shader_key key_mask;
764
765
};

766
767
768
769
770
/**
 * In order to use the same cmdstream, in particular constlen setup and const
 * emit, for both binning and draw pass (a6xx+), the binning pass re-uses it's
 * corresponding draw pass shaders const_state.
 */
771
772
773
static inline struct ir3_const_state *
ir3_const_state(const struct ir3_shader_variant *v)
{
774
775
776
   if (v->binning_pass)
      return v->nonbinning->const_state;
   return v->const_state;
777
778
}

779
780
781
782
783
784
/* Given a variant, calculate the maximum constlen it can have.
 */

static inline unsigned
ir3_max_const(const struct ir3_shader_variant *v)
{
785
786
   const struct ir3_compiler *compiler = v->shader->compiler;

787
788
   if ((v->shader->type == MESA_SHADER_COMPUTE) ||
       (v->shader->type == MESA_SHADER_KERNEL)) {
789
790
791
792
793
794
795
796
      return compiler->max_const_compute;
   } else if (v->key.safe_constlen) {
      return compiler->max_const_safe;
   } else if (v->shader->type == MESA_SHADER_FRAGMENT) {
      return compiler->max_const_frag;
   } else {
      return compiler->max_const_geom;
   }
797
798
}

799
800
801
802
803
804
805
806
807
void *ir3_shader_assemble(struct ir3_shader_variant *v);
struct ir3_shader_variant *
ir3_shader_get_variant(struct ir3_shader *shader,
                       const struct ir3_shader_key *key, bool binning_pass,
                       bool keep_ir, bool *created);
struct ir3_shader *
ir3_shader_from_nir(struct ir3_compiler *compiler, nir_shader *nir,
                    unsigned reserved_user_consts,
                    struct ir3_stream_output_info *stream_output);
Connor Abbott's avatar
Connor Abbott committed
808
uint32_t ir3_trim_constlen(struct ir3_shader_variant **variants,
809
                           const struct ir3_compiler *compiler);
810
void ir3_shader_destroy(struct ir3_shader *shader);
811
void ir3_shader_disasm(struct ir3_shader_variant *so, uint32_t *bin, FILE *out);
812
uint64_t ir3_shader_outputs(const struct ir3_shader *so);
813

814
int ir3_glsl_type_size(const struct glsl_type *type, bool bindless);
815

Rob Clark's avatar
Rob Clark committed
816
817
818
819
/*
 * Helper/util:
 */

820
821
822
823
824
/* clears shader-key flags which don't apply to the given shader.
 */
static inline void
ir3_key_clear_unused(struct ir3_shader_key *key, struct ir3_shader *shader)
{
825
826
827
828
829
   uint32_t *key_bits = (uint32_t *)key;
   uint32_t *key_mask = (uint32_t *)&shader->key_mask;
   STATIC_ASSERT(sizeof(*key) % 4 == 0);
   for (int i = 0; i < sizeof(*key) >> 2; i++)
      key_bits[i] &= key_mask[i];
830
831
}

Rob Clark's avatar
Rob Clark committed
832
static inline int
833
ir3_find_output(const struct ir3_shader_variant *so, gl_varying_slot slot)
Rob Clark's avatar
Rob Clark committed
834
{
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
   int j;

   for (j = 0; j < so->outputs_count; j++)
      if (so->outputs[j].slot == slot)
         return j;

   /* it seems optional to have a OUT.BCOLOR[n] for each OUT.COLOR[n]
    * in the vertex shader.. but the fragment shader doesn't know this
    * so  it will always have both IN.COLOR[n] and IN.BCOLOR[n].  So
    * at link time if there is no matching OUT.BCOLOR[n], we must map
    * OUT.COLOR[n] to IN.BCOLOR[n].  And visa versa if there is only
    * a OUT.BCOLOR[n] but no matching OUT.COLOR[n]
    */
   if (slot == VARYING_SLOT_BFC0) {
      slot = VARYING_SLOT_COL0;
   } else if (slot == VARYING_SLOT_BFC1) {
      slot = VARYING_SLOT_COL1;
   } else if (slot == VARYING_SLOT_COL0) {
      slot = VARYING_SLOT_BFC0;
   } else if (slot == VARYING_SLOT_COL1) {
      slot = VARYING_SLOT_BFC1;
   } else {
      return -1;
   }

   for (j = 0; j < so->outputs_count; j++)
      if (so->outputs[j].slot == slot)
         return j;

   debug_assert(0);

   return -1;
Rob Clark's avatar
Rob Clark committed
867
868
869
870
871
}

static inline int
ir3_next_varying(const struct ir3_shader_variant *so, int i)
{
872
873
874
875
   while (++i < so->inputs_count)
      if (so->inputs[i].compmask && so->inputs[i].bary)
         break;
   return i;
Rob Clark's avatar
Rob Clark committed
876
877
}

878
struct ir3_shader_linkage {
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
   /* Maximum location either consumed by the fragment shader or produced by
    * the last geometry stage, i.e. the size required for each vertex in the
    * VPC in DWORD's.
    */
   uint8_t max_loc;

   /* Number of entries in var. */
   uint8_t cnt;

   /* Bitset of locations used, including ones which are only used by the FS.
    */
   uint32_t varmask[4];

   /* Map from VS output to location. */
   struct {
      uint8_t regid;
      uint8_t compmask;
      uint8_t loc;
   } var[32];

   /* location for fixed-function gl_PrimitiveID passthrough */
   uint8_t primid_loc;

   /* location for fixed-function gl_ViewIndex passthrough */
   uint8_t viewid_loc;

   /* location for combined clip/cull distance arrays */
   uint8_t clip0_loc, clip1_loc;
907
908
909
};

static inline void
910
911
ir3_link_add(struct ir3_shader_linkage *l, uint8_t regid_, uint8_t compmask,
             uint8_t loc)
912
{
913
914
915
916
   for (int j = 0; j < util_last_bit(compmask); j++) {
      uint8_t comploc = loc + j;
      l->varmask[comploc / 32] |= 1 << (comploc % 32);
   }
917

918
   l->max_loc = MAX2(l->max_loc, loc + util_last_bit(compmask));
919

920
921
922
   if (regid_ != regid(63, 0)) {
      int i = l->cnt++;
      debug_assert(i < ARRAY_SIZE(l->var));
923

924
925
926
927
      l->var[i].regid = regid_;
      l->var[i].compmask = compmask;
      l->var[i].loc = loc;
   }
928
929
930
931
}

static inline void
ir3_link_shaders(struct ir3_shader_linkage *l,
932
933
                 const struct ir3_shader_variant *vs,
                 const struct ir3_shader_variant *fs, bool pack_vs_out)
934
{
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
   /* On older platforms, varmask isn't programmed at all, and it appears
    * that the hardware generates a mask of used VPC locations using the VS
    * output map, and hangs if a FS bary instruction references a location
    * not in the list. This means that we need to have a dummy entry in the
    * VS out map for things like gl_PointCoord which aren't written by the
    * VS. Furthermore we can't use r63.x, so just pick a random register to
    * use if there is no VS output.
    */
   const unsigned default_regid = pack_vs_out ? regid(63, 0) : regid(0, 0);
   int j = -1, k;

   l->primid_loc = 0xff;
   l->viewid_loc = 0xff;
   l->clip0_loc = 0xff;
   l->clip1_loc = 0xff;

   while (l->cnt < ARRAY_SIZE(l->var)) {
      j = ir3_next_varying(fs, j);

      if (j >= fs->inputs_count)
         break;

      if (fs->inputs[j].inloc >= fs->total_in)
         continue;

      k = ir3_find_output(vs, fs->inputs[j].slot);

      if (k < 0 && fs->inputs[j].slot == VARYING_SLOT_PRIMITIVE_ID) {
         l->primid_loc = fs->inputs[j].inloc;
      }

      if (fs->inputs[j].slot == VARYING_SLOT_VIEW_INDEX) {
         assert(k < 0);
         l->viewid_loc = fs->inputs[j].inloc;
      }

      if (fs->inputs[j].slot == VARYING_SLOT_CLIP_DIST0)
         l->clip0_loc = fs->inputs[j].inloc;

      if (fs->inputs[j].slot == VARYING_SLOT_CLIP_DIST1)
         l->clip1_loc = fs->inputs[j].inloc;

      ir3_link_add(l, k >= 0 ? vs->outputs[k].regid : default_regid,
                   fs->inputs[j].compmask, fs->inputs[j].inloc);
   }
980
981
}

Rob Clark's avatar
Rob Clark committed
982
static inline uint32_t
983
ir3_find_output_regid(const struct ir3_shader_variant *so, unsigned slot)
Rob Clark's avatar
Rob Clark committed
984
{
985
986
987
988
989
990
991
992
993
   int j;
   for (j = 0; j < so->outputs_count; j++)
      if (so->outputs[j].slot == slot) {
         uint32_t regid = so->outputs[j].regid;
         if (so->outputs[j].half)
            regid |= HALF_REG_ID;
         return regid;
      }
   return regid(63, 0);
Rob Clark's avatar
Rob Clark committed
994
995
}

996
997
void ir3_link_stream_out(struct ir3_shader_linkage *l,
                         const struct ir3_shader_variant *v);
998

999
1000
1001
#define VARYING_SLOT_GS_HEADER_IR3       (VARYING_SLOT_MAX + 0)
#define VARYING_SLOT_GS_VERTEX_FLAGS_IR3 (VARYING_SLOT_MAX + 1)
#define VARYING_SLOT_TCS_HEADER_IR3      (VARYING_SLOT_MAX + 2)
1002
#define VARYING_SLOT_REL_PATCH_ID_IR3    (VARYING_SLOT_MAX + 3)
1003

1004
1005
1006
static inline uint32_t
ir3_find_sysval_regid(const struct ir3_shader_variant *so, unsigned slot)
{
1007
1008
1009
1010
1011
   int j;
   for (j = 0; j < so->inputs_count; j++)
      if (so->inputs[j].sysval && (so->inputs[j].slot == slot))
         return so->inputs[j].regid;
   return regid(63, 0);
1012
1013
}

Rob Clark's avatar
Rob Clark committed
1014
1015
1016
1017
1018
1019
/* calculate register footprint in terms of half-regs (ie. one full
 * reg counts as two half-regs).
 */
static inline uint32_t
ir3_shader_halfregs(const struct ir3_shader_variant *v)
{
1020
   return (2 * (v->info.max_reg + 1)) + (v->info.max_half_reg + 1);
Rob Clark's avatar
Rob Clark committed
1021
1022
}

1023
1024
1025
static inline uint32_t
ir3_shader_nibo(const struct ir3_shader_variant *v)
{
1026
1027
1028
   /* The dummy variant used in binning mode won't have an actual shader. */
   if (!v->shader)
      return 0;
1029

1030
   return v->shader->nir->info.num_ssbos + v->shader->nir->info.num_images;
1031
1032
}

1033
1034
1035
static inline uint32_t
ir3_shader_branchstack_hw(const struct ir3_shader_variant *v)
{
1036
1037
1038
1039
   /* Dummy shader */
   if (!v->shader)
      return 0;

1040
   if (v->shader->compiler->gen < 5)
1041
1042
1043
1044
1045
1046
1047
1048
      return v->branchstack;

   if (v->branchstack > 0) {
      uint32_t branchstack = v->branchstack / 2 + 1;
      return MIN2(branchstack, v->shader->compiler->branchstack_size / 2);
   } else {
      return 0;
   }
1049
1050
}

1051
#endif /* IR3_SHADER_H_ */