aco_instruction_selection_setup.cpp 41.3 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
/*
 * Copyright © 2018 Valve Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 */

25
#include <array>
26
27
28
29
30
#include <unordered_map>
#include "aco_ir.h"
#include "nir.h"
#include "vulkan/radv_shader.h"
#include "vulkan/radv_descriptor_set.h"
31
#include "vulkan/radv_shader_args.h"
32
33
#include "sid.h"
#include "ac_exp_param.h"
34
#include "ac_shader_util.h"
35
36
37
38
39
40
41

#include "util/u_math.h"

#define MAX_INLINE_PUSH_CONSTS 8

namespace aco {

42
struct output_state {
43
44
45
46
47
   uint8_t mask[VARYING_SLOT_VAR31 + 1];
   Temp outputs[VARYING_SLOT_VAR31 + 1][4];
};

struct isel_context {
48
   const struct radv_nir_compiler_options *options;
49
   struct radv_shader_args *args;
50
51
52
53
54
55
   Program *program;
   nir_shader *shader;
   uint32_t constant_data_offset;
   Block *block;
   bool *divergent_vals;
   std::unique_ptr<Temp[]> allocated;
56
   std::unordered_map<unsigned, std::array<Temp,NIR_MAX_VEC_COMPONENTS>> allocated_vec;
57
   Stage stage; /* Stage */
58
   bool has_gfx10_wave64_bpermute = false;
59
60
61
62
63
64
65
66
67
68
69
70
71
   struct {
      bool has_branch;
      uint16_t loop_nest_depth = 0;
      struct {
         unsigned header_idx;
         Block* exit;
         bool has_divergent_continue = false;
         bool has_divergent_branch = false;
      } parent_loop;
      struct {
         bool is_divergent = false;
      } parent_if;
      bool exec_potentially_empty = false;
72
      std::unique_ptr<unsigned[]> nir_to_aco; /* NIR block index to ACO block index */
73
74
   } cf_info;

75
76
   Temp arg_temps[AC_MAX_ARGS];

77
   /* FS inputs */
78
   Temp persp_centroid, linear_centroid;
79

Rhys Perry's avatar
Rhys Perry committed
80
81
82
   /* GS inputs */
   Temp gs_wave_id;

83
84
85
86
   /* gathered information */
   uint64_t input_masks[MESA_SHADER_COMPUTE];
   uint64_t output_masks[MESA_SHADER_COMPUTE];

87
   /* VS output information */
Rhys Perry's avatar
Rhys Perry committed
88
   bool export_clip_dists;
89
90
   unsigned num_clip_distances;
   unsigned num_cull_distances;
91

92
93
   /* VS, FS or GS output information */
   output_state outputs;
94
95
};

96
97
98
99
100
101
102
Temp get_arg(isel_context *ctx, struct ac_arg arg)
{
   assert(arg.used);
   return ctx->arg_temps[arg.arg_index];
}

unsigned get_interp_input(nir_intrinsic_op intrin, enum glsl_interp_mode interp)
103
104
105
106
107
108
109
{
   switch (interp) {
   case INTERP_MODE_SMOOTH:
   case INTERP_MODE_NONE:
      if (intrin == nir_intrinsic_load_barycentric_pixel ||
          intrin == nir_intrinsic_load_barycentric_at_sample ||
          intrin == nir_intrinsic_load_barycentric_at_offset)
110
         return S_0286CC_PERSP_CENTER_ENA(1);
111
      else if (intrin == nir_intrinsic_load_barycentric_centroid)
112
         return S_0286CC_PERSP_CENTROID_ENA(1);
113
      else if (intrin == nir_intrinsic_load_barycentric_sample)
114
         return S_0286CC_PERSP_SAMPLE_ENA(1);
115
116
117
      break;
   case INTERP_MODE_NOPERSPECTIVE:
      if (intrin == nir_intrinsic_load_barycentric_pixel)
118
         return S_0286CC_LINEAR_CENTER_ENA(1);
119
      else if (intrin == nir_intrinsic_load_barycentric_centroid)
120
         return S_0286CC_LINEAR_CENTROID_ENA(1);
121
      else if (intrin == nir_intrinsic_load_barycentric_sample)
122
         return S_0286CC_LINEAR_SAMPLE_ENA(1);
123
124
125
126
      break;
   default:
      break;
   }
127
   return 0;
128
129
130
131
132
}

void init_context(isel_context *ctx, nir_shader *shader)
{
   nir_function_impl *impl = nir_shader_get_entrypoint(shader);
133
   unsigned lane_mask_size = ctx->program->lane_mask.size();
134
135
136
137
138

   ctx->shader = shader;
   ctx->divergent_vals = nir_divergence_analysis(shader, nir_divergence_view_index_uniform);

   std::unique_ptr<Temp[]> allocated{new Temp[impl->ssa_alloc]()};
139
140

   unsigned spi_ps_inputs = 0;
141

142
143
   std::unique_ptr<unsigned[]> nir_to_aco{new unsigned[impl->num_blocks]()};

144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
   bool done = false;
   while (!done) {
      done = true;
      nir_foreach_block(block, impl) {
         nir_foreach_instr(instr, block) {
            switch(instr->type) {
            case nir_instr_type_alu: {
               nir_alu_instr *alu_instr = nir_instr_as_alu(instr);
               unsigned size =  alu_instr->dest.dest.ssa.num_components;
               if (alu_instr->dest.dest.ssa.bit_size == 64)
                  size *= 2;
               RegType type = RegType::sgpr;
               switch(alu_instr->op) {
                  case nir_op_fmul:
                  case nir_op_fadd:
                  case nir_op_fsub:
                  case nir_op_fmax:
                  case nir_op_fmin:
                  case nir_op_fmax3:
                  case nir_op_fmin3:
                  case nir_op_fmed3:
                  case nir_op_fneg:
                  case nir_op_fabs:
                  case nir_op_fsat:
                  case nir_op_fsign:
                  case nir_op_frcp:
                  case nir_op_frsq:
                  case nir_op_fsqrt:
                  case nir_op_fexp2:
                  case nir_op_flog2:
                  case nir_op_ffract:
                  case nir_op_ffloor:
                  case nir_op_fceil:
                  case nir_op_ftrunc:
                  case nir_op_fround_even:
                  case nir_op_fsin:
                  case nir_op_fcos:
                  case nir_op_f2f32:
                  case nir_op_f2f64:
                  case nir_op_u2f32:
                  case nir_op_u2f64:
                  case nir_op_i2f32:
                  case nir_op_i2f64:
                  case nir_op_pack_half_2x16:
                  case nir_op_unpack_half_2x16_split_x:
                  case nir_op_unpack_half_2x16_split_y:
                  case nir_op_fddx:
                  case nir_op_fddy:
                  case nir_op_fddx_fine:
                  case nir_op_fddy_fine:
                  case nir_op_fddx_coarse:
                  case nir_op_fddy_coarse:
                  case nir_op_fquantize2f16:
                  case nir_op_ldexp:
                  case nir_op_frexp_sig:
                  case nir_op_frexp_exp:
                  case nir_op_cube_face_index:
                  case nir_op_cube_face_coord:
                     type = RegType::vgpr;
                     break;
                  case nir_op_flt:
                  case nir_op_fge:
                  case nir_op_feq:
                  case nir_op_fne:
                  case nir_op_ilt:
                  case nir_op_ige:
                  case nir_op_ult:
                  case nir_op_uge:
                  case nir_op_ieq:
                  case nir_op_ine:
                  case nir_op_i2b1:
215
                     size = lane_mask_size;
216
217
218
219
220
221
222
223
224
225
226
                     break;
                  case nir_op_f2i64:
                  case nir_op_f2u64:
                  case nir_op_b2i32:
                  case nir_op_b2f32:
                  case nir_op_f2i32:
                  case nir_op_f2u32:
                     type = ctx->divergent_vals[alu_instr->dest.dest.ssa.index] ? RegType::vgpr : RegType::sgpr;
                     break;
                  case nir_op_bcsel:
                     if (alu_instr->dest.dest.ssa.bit_size == 1) {
227
                        size = lane_mask_size;
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
                     } else {
                        if (ctx->divergent_vals[alu_instr->dest.dest.ssa.index]) {
                           type = RegType::vgpr;
                        } else {
                           if (allocated[alu_instr->src[1].src.ssa->index].type() == RegType::vgpr ||
                               allocated[alu_instr->src[2].src.ssa->index].type() == RegType::vgpr) {
                              type = RegType::vgpr;
                           }
                        }
                        if (alu_instr->src[1].src.ssa->num_components == 1 && alu_instr->src[2].src.ssa->num_components == 1) {
                           assert(allocated[alu_instr->src[1].src.ssa->index].size() == allocated[alu_instr->src[2].src.ssa->index].size());
                           size = allocated[alu_instr->src[1].src.ssa->index].size();
                        }
                     }
                     break;
                  case nir_op_mov:
                     if (alu_instr->dest.dest.ssa.bit_size == 1) {
245
                        size = lane_mask_size;
246
247
248
249
250
251
                     } else {
                        type = ctx->divergent_vals[alu_instr->dest.dest.ssa.index] ? RegType::vgpr : RegType::sgpr;
                     }
                     break;
                  default:
                     if (alu_instr->dest.dest.ssa.bit_size == 1) {
252
                        size = lane_mask_size;
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
                     } else {
                        for (unsigned i = 0; i < nir_op_infos[alu_instr->op].num_inputs; i++) {
                           if (allocated[alu_instr->src[i].src.ssa->index].type() == RegType::vgpr)
                              type = RegType::vgpr;
                        }
                     }
                     break;
               }
               allocated[alu_instr->dest.dest.ssa.index] = Temp(0, RegClass(type, size));
               break;
            }
            case nir_instr_type_load_const: {
               unsigned size = nir_instr_as_load_const(instr)->def.num_components;
               if (nir_instr_as_load_const(instr)->def.bit_size == 64)
                  size *= 2;
268
               else if (nir_instr_as_load_const(instr)->def.bit_size == 1)
269
                  size *= lane_mask_size;
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
               allocated[nir_instr_as_load_const(instr)->def.index] = Temp(0, RegClass(RegType::sgpr, size));
               break;
            }
            case nir_instr_type_intrinsic: {
               nir_intrinsic_instr *intrinsic = nir_instr_as_intrinsic(instr);
               if (!nir_intrinsic_infos[intrinsic->intrinsic].has_dest)
                  break;
               unsigned size =  intrinsic->dest.ssa.num_components;
               if (intrinsic->dest.ssa.bit_size == 64)
                  size *= 2;
               RegType type = RegType::sgpr;
               switch(intrinsic->intrinsic) {
                  case nir_intrinsic_load_push_constant:
                  case nir_intrinsic_load_work_group_id:
                  case nir_intrinsic_load_num_work_groups:
                  case nir_intrinsic_load_subgroup_id:
                  case nir_intrinsic_load_num_subgroups:
                  case nir_intrinsic_load_first_vertex:
                  case nir_intrinsic_load_base_instance:
                  case nir_intrinsic_get_buffer_size:
                  case nir_intrinsic_vote_all:
                  case nir_intrinsic_vote_any:
                  case nir_intrinsic_read_first_invocation:
                  case nir_intrinsic_read_invocation:
                  case nir_intrinsic_first_invocation:
                     type = RegType::sgpr;
296
                     if (intrinsic->dest.ssa.bit_size == 1)
297
                        size = lane_mask_size;
298
299
300
301
302
303
304
                     break;
                  case nir_intrinsic_ballot:
                     type = RegType::sgpr;
                     break;
                  case nir_intrinsic_load_sample_id:
                  case nir_intrinsic_load_sample_mask_in:
                  case nir_intrinsic_load_input:
305
                  case nir_intrinsic_load_per_vertex_input:
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
                  case nir_intrinsic_load_vertex_id:
                  case nir_intrinsic_load_vertex_id_zero_base:
                  case nir_intrinsic_load_barycentric_sample:
                  case nir_intrinsic_load_barycentric_pixel:
                  case nir_intrinsic_load_barycentric_centroid:
                  case nir_intrinsic_load_barycentric_at_sample:
                  case nir_intrinsic_load_barycentric_at_offset:
                  case nir_intrinsic_load_interpolated_input:
                  case nir_intrinsic_load_frag_coord:
                  case nir_intrinsic_load_sample_pos:
                  case nir_intrinsic_load_layer_id:
                  case nir_intrinsic_load_local_invocation_id:
                  case nir_intrinsic_load_local_invocation_index:
                  case nir_intrinsic_load_subgroup_invocation:
                  case nir_intrinsic_write_invocation_amd:
                  case nir_intrinsic_mbcnt_amd:
                  case nir_intrinsic_load_instance_id:
                  case nir_intrinsic_ssbo_atomic_add:
                  case nir_intrinsic_ssbo_atomic_imin:
                  case nir_intrinsic_ssbo_atomic_umin:
                  case nir_intrinsic_ssbo_atomic_imax:
                  case nir_intrinsic_ssbo_atomic_umax:
                  case nir_intrinsic_ssbo_atomic_and:
                  case nir_intrinsic_ssbo_atomic_or:
                  case nir_intrinsic_ssbo_atomic_xor:
                  case nir_intrinsic_ssbo_atomic_exchange:
                  case nir_intrinsic_ssbo_atomic_comp_swap:
Rhys Perry's avatar
Rhys Perry committed
333
334
335
336
337
338
339
340
341
342
                  case nir_intrinsic_global_atomic_add:
                  case nir_intrinsic_global_atomic_imin:
                  case nir_intrinsic_global_atomic_umin:
                  case nir_intrinsic_global_atomic_imax:
                  case nir_intrinsic_global_atomic_umax:
                  case nir_intrinsic_global_atomic_and:
                  case nir_intrinsic_global_atomic_or:
                  case nir_intrinsic_global_atomic_xor:
                  case nir_intrinsic_global_atomic_exchange:
                  case nir_intrinsic_global_atomic_comp_swap:
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
                  case nir_intrinsic_image_deref_atomic_add:
                  case nir_intrinsic_image_deref_atomic_umin:
                  case nir_intrinsic_image_deref_atomic_imin:
                  case nir_intrinsic_image_deref_atomic_umax:
                  case nir_intrinsic_image_deref_atomic_imax:
                  case nir_intrinsic_image_deref_atomic_and:
                  case nir_intrinsic_image_deref_atomic_or:
                  case nir_intrinsic_image_deref_atomic_xor:
                  case nir_intrinsic_image_deref_atomic_exchange:
                  case nir_intrinsic_image_deref_atomic_comp_swap:
                  case nir_intrinsic_image_deref_size:
                  case nir_intrinsic_shared_atomic_add:
                  case nir_intrinsic_shared_atomic_imin:
                  case nir_intrinsic_shared_atomic_umin:
                  case nir_intrinsic_shared_atomic_imax:
                  case nir_intrinsic_shared_atomic_umax:
                  case nir_intrinsic_shared_atomic_and:
                  case nir_intrinsic_shared_atomic_or:
                  case nir_intrinsic_shared_atomic_xor:
                  case nir_intrinsic_shared_atomic_exchange:
                  case nir_intrinsic_shared_atomic_comp_swap:
                  case nir_intrinsic_load_scratch:
365
366
                  case nir_intrinsic_load_invocation_id:
                  case nir_intrinsic_load_primitive_id:
367
368
369
370
371
372
373
374
375
376
377
                     type = RegType::vgpr;
                     break;
                  case nir_intrinsic_shuffle:
                  case nir_intrinsic_quad_broadcast:
                  case nir_intrinsic_quad_swap_horizontal:
                  case nir_intrinsic_quad_swap_vertical:
                  case nir_intrinsic_quad_swap_diagonal:
                  case nir_intrinsic_quad_swizzle_amd:
                  case nir_intrinsic_masked_swizzle_amd:
                  case nir_intrinsic_inclusive_scan:
                  case nir_intrinsic_exclusive_scan:
378
                     if (intrinsic->dest.ssa.bit_size == 1) {
379
                        size = lane_mask_size;
380
                        type = RegType::sgpr;
381
                     } else if (!ctx->divergent_vals[intrinsic->dest.ssa.index]) {
382
383
384
385
386
387
388
389
390
391
392
393
                        type = RegType::sgpr;
                     } else {
                        type = RegType::vgpr;
                     }
                     break;
                  case nir_intrinsic_load_view_index:
                     type = ctx->stage == fragment_fs ? RegType::vgpr : RegType::sgpr;
                     break;
                  case nir_intrinsic_load_front_face:
                  case nir_intrinsic_load_helper_invocation:
                  case nir_intrinsic_is_helper_invocation:
                     type = RegType::sgpr;
394
                     size = lane_mask_size;
395
396
                     break;
                  case nir_intrinsic_reduce:
397
                     if (intrinsic->dest.ssa.bit_size == 1) {
398
                        size = lane_mask_size;
399
                        type = RegType::sgpr;
Timur Kristóf's avatar
Timur Kristóf committed
400
                     } else if (!ctx->divergent_vals[intrinsic->dest.ssa.index]) {
401
402
403
404
405
406
407
408
                        type = RegType::sgpr;
                     } else {
                        type = RegType::vgpr;
                     }
                     break;
                  case nir_intrinsic_load_ubo:
                  case nir_intrinsic_load_ssbo:
                  case nir_intrinsic_load_global:
409
                  case nir_intrinsic_vulkan_resource_index:
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
                     type = ctx->divergent_vals[intrinsic->dest.ssa.index] ? RegType::vgpr : RegType::sgpr;
                     break;
                  /* due to copy propagation, the swizzled imov is removed if num dest components == 1 */
                  case nir_intrinsic_load_shared:
                     if (ctx->divergent_vals[intrinsic->dest.ssa.index])
                        type = RegType::vgpr;
                     else
                        type = RegType::sgpr;
                     break;
                  default:
                     for (unsigned i = 0; i < nir_intrinsic_infos[intrinsic->intrinsic].num_srcs; i++) {
                        if (allocated[intrinsic->src[i].ssa->index].type() == RegType::vgpr)
                           type = RegType::vgpr;
                     }
                     break;
               }
               allocated[intrinsic->dest.ssa.index] = Temp(0, RegClass(type, size));

               switch(intrinsic->intrinsic) {
                  case nir_intrinsic_load_barycentric_sample:
                  case nir_intrinsic_load_barycentric_pixel:
                  case nir_intrinsic_load_barycentric_centroid:
                  case nir_intrinsic_load_barycentric_at_sample:
                  case nir_intrinsic_load_barycentric_at_offset: {
                     glsl_interp_mode mode = (glsl_interp_mode)nir_intrinsic_interp_mode(intrinsic);
435
                     spi_ps_inputs |= get_interp_input(intrinsic->intrinsic, mode);
436
437
438
                     break;
                  }
                  case nir_intrinsic_load_front_face:
439
                     spi_ps_inputs |= S_0286CC_FRONT_FACE_ENA(1);
440
441
442
443
444
445
                     break;
                  case nir_intrinsic_load_frag_coord:
                  case nir_intrinsic_load_sample_pos: {
                     uint8_t mask = nir_ssa_def_components_read(&intrinsic->dest.ssa);
                     for (unsigned i = 0; i < 4; i++) {
                        if (mask & (1 << i))
446
                           spi_ps_inputs |= S_0286CC_POS_X_FLOAT_ENA(1) << i;
447
448
449
450
451

                     }
                     break;
                  }
                  case nir_intrinsic_load_sample_id:
452
                     spi_ps_inputs |= S_0286CC_ANCILLARY_ENA(1);
453
454
                     break;
                  case nir_intrinsic_load_sample_mask_in:
455
456
                     spi_ps_inputs |= S_0286CC_ANCILLARY_ENA(1);
                     spi_ps_inputs |= S_0286CC_SAMPLE_COVERAGE_ENA(1);
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
                     break;
                  default:
                     break;
               }
               break;
            }
            case nir_instr_type_tex: {
               nir_tex_instr* tex = nir_instr_as_tex(instr);
               unsigned size = tex->dest.ssa.num_components;

               if (tex->dest.ssa.bit_size == 64)
                  size *= 2;
               if (tex->op == nir_texop_texture_samples)
                  assert(!ctx->divergent_vals[tex->dest.ssa.index]);
               if (ctx->divergent_vals[tex->dest.ssa.index])
                  allocated[tex->dest.ssa.index] = Temp(0, RegClass(RegType::vgpr, size));
               else
                  allocated[tex->dest.ssa.index] = Temp(0, RegClass(RegType::sgpr, size));
               break;
            }
            case nir_instr_type_parallel_copy: {
               nir_foreach_parallel_copy_entry(entry, nir_instr_as_parallel_copy(instr)) {
                  allocated[entry->dest.ssa.index] = allocated[entry->src.ssa->index];
               }
               break;
            }
            case nir_instr_type_ssa_undef: {
               unsigned size = nir_instr_as_ssa_undef(instr)->def.num_components;
               if (nir_instr_as_ssa_undef(instr)->def.bit_size == 64)
                  size *= 2;
               allocated[nir_instr_as_ssa_undef(instr)->def.index] = Temp(0, RegClass(RegType::sgpr, size));
               break;
            }
            case nir_instr_type_phi: {
               nir_phi_instr* phi = nir_instr_as_phi(instr);
               RegType type;
               unsigned size = phi->dest.ssa.num_components;

               if (phi->dest.ssa.bit_size == 1) {
                  assert(size == 1 && "multiple components not yet supported on boolean phis.");
                  type = RegType::sgpr;
498
                  size *= lane_mask_size;
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
                  allocated[phi->dest.ssa.index] = Temp(0, RegClass(type, size));
                  break;
               }

               if (ctx->divergent_vals[phi->dest.ssa.index]) {
                  type = RegType::vgpr;
               } else {
                  type = RegType::sgpr;
                  nir_foreach_phi_src (src, phi) {
                     if (allocated[src->src.ssa->index].type() == RegType::vgpr)
                        type = RegType::vgpr;
                     if (allocated[src->src.ssa->index].type() == RegType::none)
                        done = false;
                  }
               }

               size *= phi->dest.ssa.bit_size == 64 ? 2 : 1;
               RegClass rc = RegClass(type, size);
               if (rc != allocated[phi->dest.ssa.index].regClass()) {
                  done = false;
               } else {
                  nir_foreach_phi_src(src, phi)
                     assert(allocated[src->src.ssa->index].size() == rc.size());
               }
               allocated[phi->dest.ssa.index] = Temp(0, rc);
               break;
            }
            default:
               break;
            }
         }
      }
   }

533
534
535
   if (G_0286CC_POS_W_FLOAT_ENA(spi_ps_inputs)) {
      /* If POS_W_FLOAT (11) is enabled, at least one of PERSP_* must be enabled too */
      spi_ps_inputs |= S_0286CC_PERSP_CENTER_ENA(1);
536
537
   }

538
539
540
   if (!(spi_ps_inputs & 0x7F)) {
      /* At least one of PERSP_* (0xF) or LINEAR_* (0x70) must be enabled */
      spi_ps_inputs |= S_0286CC_PERSP_CENTER_ENA(1);
541
542
   }

543
544
   ctx->program->config->spi_ps_input_ena = spi_ps_inputs;
   ctx->program->config->spi_ps_input_addr = spi_ps_inputs;
545

546
547
   for (unsigned i = 0; i < impl->ssa_alloc; i++)
      allocated[i] = Temp(ctx->program->allocateId(), allocated[i].regClass());
548

549
   ctx->allocated.reset(allocated.release());
550
   ctx->cf_info.nir_to_aco.reset(nir_to_aco.release());
551
552
}

553
Pseudo_instruction *add_startpgm(struct isel_context *ctx)
554
{
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
   unsigned arg_count = ctx->args->ac.arg_count;
   if (ctx->stage == fragment_fs) {
      /* LLVM optimizes away unused FS inputs and computes spi_ps_input_addr
       * itself and then communicates the results back via the ELF binary.
       * Mirror what LLVM does by re-mapping the VGPR arguments here.
       *
       * TODO: If we made the FS input scanning code into a separate pass that
       * could run before argument setup, then this wouldn't be necessary
       * anymore.
       */
      struct ac_shader_args *args = &ctx->args->ac;
      arg_count = 0;
      for (unsigned i = 0, vgpr_arg = 0, vgpr_reg = 0; i < args->arg_count; i++) {
         if (args->args[i].file != AC_ARG_VGPR) {
            arg_count++;
            continue;
         }
572

573
574
575
576
577
578
579
580
         if (!(ctx->program->config->spi_ps_input_addr & (1 << vgpr_arg))) {
            args->args[i].skip = true;
         } else {
            args->args[i].offset = vgpr_reg;
            vgpr_reg += args->args[i].size;
            arg_count++;
         }
         vgpr_arg++;
581
582
583
      }
   }

584
585
586
587
   aco_ptr<Pseudo_instruction> startpgm{create_instruction<Pseudo_instruction>(aco_opcode::p_startpgm, Format::PSEUDO, 0, arg_count + 1)};
   for (unsigned i = 0, arg = 0; i < ctx->args->ac.arg_count; i++) {
      if (ctx->args->ac.args[i].skip)
         continue;
588

589
590
591
592
593
594
595
596
597
598
      enum ac_arg_regfile file = ctx->args->ac.args[i].file;
      unsigned size = ctx->args->ac.args[i].size;
      unsigned reg = ctx->args->ac.args[i].offset;
      RegClass type = RegClass(file == AC_ARG_SGPR ? RegType::sgpr : RegType::vgpr, size);
      Temp dst = Temp{ctx->program->allocateId(), type};
      ctx->arg_temps[i] = dst;
      startpgm->definitions[arg] = Definition(dst);
      startpgm->definitions[arg].setFixed(PhysReg{file == AC_ARG_SGPR ? reg : reg + 256});
      arg++;
   }
599
   startpgm->definitions[arg_count] = Definition{ctx->program->allocateId(), exec, ctx->program->lane_mask};
600
   Pseudo_instruction *instr = startpgm.get();
601
   ctx->block->instructions.push_back(std::move(startpgm));
602

603
604
605
606
607
608
   /* Stash these in the program so that they can be accessed later when
    * handling spilling.
    */
   ctx->program->private_segment_buffer = get_arg(ctx, ctx->args->ring_offsets);
   ctx->program->scratch_offset = get_arg(ctx, ctx->args->scratch_offset);

609
   return instr;
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
}

int
type_size(const struct glsl_type *type, bool bindless)
{
   // TODO: don't we need type->std430_base_alignment() here?
   return glsl_count_attribute_slots(type, false);
}

void
shared_var_info(const struct glsl_type *type, unsigned *size, unsigned *align)
{
   assert(glsl_type_is_vector_or_scalar(type));

   uint32_t comp_size = glsl_type_is_boolean(type)
      ? 4 : glsl_get_bit_size(type) / 8;
   unsigned length = glsl_get_vector_elements(type);
   *size = comp_size * length,
   *align = comp_size;
}

Rhys Perry's avatar
Rhys Perry committed
631
632
633
634
static bool
mem_vectorize_callback(unsigned align, unsigned bit_size,
                       unsigned num_components, unsigned high_offset,
                       nir_intrinsic_instr *low, nir_intrinsic_instr *high)
635
{
Rhys Perry's avatar
Rhys Perry committed
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
   if ((bit_size != 32 && bit_size != 64) || num_components > 4)
      return false;

   /* >128 bit loads are split except with SMEM */
   if (bit_size * num_components > 128)
      return false;

   switch (low->intrinsic) {
   case nir_intrinsic_load_ubo:
   case nir_intrinsic_load_ssbo:
   case nir_intrinsic_store_ssbo:
   case nir_intrinsic_load_push_constant:
      return align % 4 == 0;
   case nir_intrinsic_load_deref:
   case nir_intrinsic_store_deref:
      assert(nir_src_as_deref(low->src[0])->mode == nir_var_mem_shared);
      /* fallthrough */
   case nir_intrinsic_load_shared:
   case nir_intrinsic_store_shared:
      if (bit_size * num_components > 64) /* 96 and 128 bit loads require 128 bit alignment and are split otherwise */
         return align % 16 == 0;
      else
         return align % 4 == 0;
659
   default:
Rhys Perry's avatar
Rhys Perry committed
660
      return false;
661
   }
Rhys Perry's avatar
Rhys Perry committed
662
   return false;
663
664
}

Rhys Perry's avatar
Rhys Perry committed
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
void
setup_vs_output_info(isel_context *ctx, nir_shader *nir,
                     bool export_prim_id, bool export_clip_dists,
                     radv_vs_output_info *outinfo)
{
   memset(outinfo->vs_output_param_offset, AC_EXP_PARAM_UNDEFINED,
          sizeof(outinfo->vs_output_param_offset));

   outinfo->param_exports = 0;
   int pos_written = 0x1;
   if (outinfo->writes_pointsize || outinfo->writes_viewport_index || outinfo->writes_layer)
      pos_written |= 1 << 1;

   uint64_t mask = ctx->output_masks[nir->info.stage];
   while (mask) {
      int idx = u_bit_scan64(&mask);
      if (idx >= VARYING_SLOT_VAR0 || idx == VARYING_SLOT_LAYER || idx == VARYING_SLOT_PRIMITIVE_ID ||
          ((idx == VARYING_SLOT_CLIP_DIST0 || idx == VARYING_SLOT_CLIP_DIST1) && export_clip_dists)) {
         if (outinfo->vs_output_param_offset[idx] == AC_EXP_PARAM_UNDEFINED)
            outinfo->vs_output_param_offset[idx] = outinfo->param_exports++;
      }
   }
   if (outinfo->writes_layer &&
       outinfo->vs_output_param_offset[VARYING_SLOT_LAYER] == AC_EXP_PARAM_UNDEFINED) {
      /* when ctx->options->key.has_multiview_view_index = true, the layer
       * variable isn't declared in NIR and it's isel's job to get the layer */
      outinfo->vs_output_param_offset[VARYING_SLOT_LAYER] = outinfo->param_exports++;
   }

   if (export_prim_id) {
      assert(outinfo->vs_output_param_offset[VARYING_SLOT_PRIMITIVE_ID] == AC_EXP_PARAM_UNDEFINED);
      outinfo->vs_output_param_offset[VARYING_SLOT_PRIMITIVE_ID] = outinfo->param_exports++;
   }

   ctx->export_clip_dists = export_clip_dists;
   ctx->num_clip_distances = util_bitcount(outinfo->clip_dist_mask);
   ctx->num_cull_distances = util_bitcount(outinfo->cull_dist_mask);

   assert(ctx->num_clip_distances + ctx->num_cull_distances <= 8);

   if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
      pos_written |= 1 << 2;
   if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
      pos_written |= 1 << 3;

   outinfo->pos_exports = util_bitcount(pos_written);
}

713
714
715
716
717
718
719
720
721
void
setup_vs_variables(isel_context *ctx, nir_shader *nir)
{
   nir_foreach_variable(variable, &nir->inputs)
   {
      variable->data.driver_location = variable->data.location * 4;
   }
   nir_foreach_variable(variable, &nir->outputs)
   {
722
723
      if (ctx->stage == vertex_geometry_gs)
         variable->data.driver_location = util_bitcount64(ctx->output_masks[nir->info.stage] & ((1ull << variable->data.location) - 1ull)) * 4;
Rhys Perry's avatar
Rhys Perry committed
724
725
726
      else if (ctx->stage == vertex_es)
         //TODO: make this more compact
         variable->data.driver_location = shader_io_get_unique_index((gl_varying_slot)variable->data.location) * 4;
727
728
      else
         variable->data.driver_location = variable->data.location * 4;
729
730
   }

731
732
   if (ctx->stage == vertex_vs) {
      radv_vs_output_info *outinfo = &ctx->program->info->vs.outinfo;
Rhys Perry's avatar
Rhys Perry committed
733
734
      setup_vs_output_info(ctx, nir, outinfo->export_prim_id,
                           ctx->options->key.vs_common_out.export_clip_dists, outinfo);
Rhys Perry's avatar
Rhys Perry committed
735
   } else if (ctx->stage == vertex_geometry_gs || ctx->stage == vertex_es) {
736
737
738
739
740
741
742
743
      /* TODO: radv_nir_shader_info_pass() already sets this but it's larger
       * than it needs to be in order to set it better, we have to improve
       * radv_nir_shader_info_pass() because gfx9_get_gs_info() uses
       * esgs_itemsize and has to be done before compilation
       */
      /* radv_es_output_info *outinfo = &ctx->program->info->vs.es_info;
      outinfo->esgs_itemsize = util_bitcount64(ctx->output_masks[nir->info.stage]) * 16u; */
   }
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
}

void
setup_variables(isel_context *ctx, nir_shader *nir)
{
   switch (nir->info.stage) {
   case MESA_SHADER_FRAGMENT: {
      nir_foreach_variable(variable, &nir->outputs)
      {
         int idx = variable->data.location + variable->data.index;
         variable->data.driver_location = idx * 4;
      }
      break;
   }
   case MESA_SHADER_COMPUTE: {
759
760
      ctx->program->config->lds_size = (nir->info.cs.shared_size + ctx->program->lds_alloc_granule - 1) /
                                       ctx->program->lds_alloc_granule;
761
762
763
764
765
766
      break;
   }
   case MESA_SHADER_VERTEX: {
      setup_vs_variables(ctx, nir);
      break;
   }
767
   case MESA_SHADER_GEOMETRY: {
Rhys Perry's avatar
Rhys Perry committed
768
769
770
771
772
773
774
775
776
777
      assert(ctx->stage == vertex_geometry_gs || ctx->stage == geometry_gs);
      if (ctx->stage == vertex_geometry_gs) {
         nir_foreach_variable(variable, &nir->inputs) {
            variable->data.driver_location = util_bitcount64(ctx->input_masks[nir->info.stage] & ((1ull << variable->data.location) - 1ull)) * 4;
         }
      } else {
         //TODO: make this more compact
         nir_foreach_variable(variable, &nir->inputs) {
            variable->data.driver_location = shader_io_get_unique_index((gl_varying_slot)variable->data.location) * 4;
         }
778
779
780
781
      }
      nir_foreach_variable(variable, &nir->outputs) {
         variable->data.driver_location = variable->data.location * 4;
      }
Rhys Perry's avatar
Rhys Perry committed
782
783
      if (ctx->stage == vertex_geometry_gs)
         ctx->program->info->gs.es_type = MESA_SHADER_VERTEX; /* tesselation shaders are not yet supported */
784
785
      break;
   }
786
787
788
789
790
   default:
      unreachable("Unhandled shader stage.");
   }
}

791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
void
get_io_masks(isel_context *ctx, unsigned shader_count, struct nir_shader *const *shaders)
{
   for (unsigned i = 0; i < shader_count; i++) {
      nir_shader *nir = shaders[i];
      if (nir->info.stage == MESA_SHADER_COMPUTE)
         continue;

      uint64_t output_mask = 0;
      nir_foreach_variable(variable, &nir->outputs) {
         const glsl_type *type = variable->type;
         if (nir_is_per_vertex_io(variable, nir->info.stage))
            type = type->fields.array;
         unsigned slots = type->count_attribute_slots(false);
         if (variable->data.compact) {
            unsigned component_count = variable->data.location_frac + type->length;
            slots = (component_count + 3) / 4;
         }
         output_mask |= ((1ull << slots) - 1) << variable->data.location;
      }

      uint64_t input_mask = 0;
      nir_foreach_variable(variable, &nir->inputs) {
         const glsl_type *type = variable->type;
         if (nir_is_per_vertex_io(variable, nir->info.stage))
            type = type->fields.array;
         unsigned slots = type->count_attribute_slots(false);
         if (variable->data.compact) {
            unsigned component_count = variable->data.location_frac + type->length;
            slots = (component_count + 3) / 4;
         }
         input_mask |= ((1ull << slots) - 1) << variable->data.location;
      }

      ctx->output_masks[nir->info.stage] |= output_mask;
      if (i + 1 < shader_count)
         ctx->input_masks[shaders[i + 1]->info.stage] |= output_mask;

      ctx->input_masks[nir->info.stage] |= input_mask;
      if (i)
         ctx->output_masks[shaders[i - 1]->info.stage] |= input_mask;
   }
}

Rhys Perry's avatar
Rhys Perry committed
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
void
setup_nir(isel_context *ctx, nir_shader *nir)
{
   Program *program = ctx->program;

   /* align and copy constant data */
   while (program->constant_data.size() % 4u)
      program->constant_data.push_back(0);
   ctx->constant_data_offset = program->constant_data.size();
   program->constant_data.insert(program->constant_data.end(),
                                 (uint8_t*)nir->constant_data,
                                 (uint8_t*)nir->constant_data + nir->constant_data_size);

   /* the variable setup has to be done before lower_io / CSE */
   setup_variables(ctx, nir);

   /* optimize and lower memory operations */
   bool lower_to_scalar = false;
   bool lower_pack = false;
   if (nir_opt_load_store_vectorize(nir,
                                    (nir_variable_mode)(nir_var_mem_ssbo | nir_var_mem_ubo |
                                                        nir_var_mem_push_const | nir_var_mem_shared),
                                    mem_vectorize_callback)) {
      lower_to_scalar = true;
      lower_pack = true;
   }
   if (nir->info.stage != MESA_SHADER_COMPUTE)
      nir_lower_io(nir, (nir_variable_mode)(nir_var_shader_in | nir_var_shader_out), type_size, (nir_lower_io_options)0);
   nir_lower_explicit_io(nir, nir_var_mem_global, nir_address_format_64bit_global);

   if (lower_to_scalar)
      nir_lower_alu_to_scalar(nir, NULL, NULL);
   if (lower_pack)
      nir_lower_pack(nir);

   /* lower ALU operations */
   // TODO: implement logic64 in aco, it's more effective for sgprs
   nir_lower_int64(nir, nir->options->lower_int64_options);

   nir_opt_idiv_const(nir, 32);
   nir_lower_idiv(nir, nir_lower_idiv_precise);

   /* optimize the lowered ALU operations */
   bool more_algebraic = true;
   while (more_algebraic) {
      more_algebraic = false;
      NIR_PASS_V(nir, nir_copy_prop);
      NIR_PASS_V(nir, nir_opt_dce);
      NIR_PASS_V(nir, nir_opt_constant_folding);
      NIR_PASS(more_algebraic, nir, nir_opt_algebraic);
   }

887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
   /* Do late algebraic optimization to turn add(a, neg(b)) back into
    * subs, then the mandatory cleanup after algebraic.  Note that it may
    * produce fnegs, and if so then we need to keep running to squash
    * fneg(fneg(a)).
    */
   bool more_late_algebraic = true;
   while (more_late_algebraic) {
      more_late_algebraic = false;
      NIR_PASS(more_late_algebraic, nir, nir_opt_algebraic_late);
      NIR_PASS_V(nir, nir_opt_constant_folding);
      NIR_PASS_V(nir, nir_copy_prop);
      NIR_PASS_V(nir, nir_opt_dce);
      NIR_PASS_V(nir, nir_opt_cse);
   }

Rhys Perry's avatar
Rhys Perry committed
902
903
904
905
   /* cleanup passes */
   nir_lower_load_const_to_scalar(nir);
   nir_opt_shrink_load(nir);
   nir_move_options move_opts = (nir_move_options)(
Rhys Perry's avatar
Rhys Perry committed
906
907
      nir_move_const_undef | nir_move_load_ubo | nir_move_load_input |
      nir_move_comparisons | nir_move_copies);
Rhys Perry's avatar
Rhys Perry committed
908
909
910
911
912
913
914
915
916
917
   nir_opt_sink(nir, move_opts);
   nir_opt_move(nir, move_opts);
   nir_convert_to_lcssa(nir, true, false);
   nir_lower_phis_to_scalar(nir);

   nir_function_impl *func = nir_shader_get_entrypoint(nir);
   nir_index_ssa_defs(func);
   nir_metadata_require(func, nir_metadata_block_index);
}

918
919
920
921
922
isel_context
setup_isel_context(Program* program,
                   unsigned shader_count,
                   struct nir_shader *const *shaders,
                   ac_shader_config* config,
Rhys Perry's avatar
Rhys Perry committed
923
924
                   struct radv_shader_args *args,
                   bool is_gs_copy_shader)
925
926
927
928
929
930
931
932
933
934
935
936
937
938
{
   program->stage = 0;
   for (unsigned i = 0; i < shader_count; i++) {
      switch (shaders[i]->info.stage) {
      case MESA_SHADER_VERTEX:
         program->stage |= sw_vs;
         break;
      case MESA_SHADER_TESS_CTRL:
         program->stage |= sw_tcs;
         break;
      case MESA_SHADER_TESS_EVAL:
         program->stage |= sw_tes;
         break;
      case MESA_SHADER_GEOMETRY:
Rhys Perry's avatar
Rhys Perry committed
939
         program->stage |= is_gs_copy_shader ? sw_gs_copy : sw_gs;
940
941
942
943
944
945
946
947
948
949
950
         break;
      case MESA_SHADER_FRAGMENT:
         program->stage |= sw_fs;
         break;
      case MESA_SHADER_COMPUTE:
         program->stage |= sw_cs;
         break;
      default:
         unreachable("Shader stage not implemented");
      }
   }
951
952
   bool gfx9_plus = args->options->chip_class >= GFX9;
   bool ngg = args->shader_info->is_ngg && args->options->chip_class >= GFX10;
Rhys Perry's avatar
Rhys Perry committed
953
954
955
   if (program->stage == sw_vs && args->shader_info->vs.as_es)
      program->stage |= hw_es;
   else if (program->stage == sw_vs && !args->shader_info->vs.as_ls)
956
      program->stage |= hw_vs;
Rhys Perry's avatar
Rhys Perry committed
957
958
   else if (program->stage == sw_gs)
      program->stage |= hw_gs;
959
960
961
962
   else if (program->stage == sw_fs)
      program->stage |= hw_fs;
   else if (program->stage == sw_cs)
      program->stage |= hw_cs;
Rhys Perry's avatar
Rhys Perry committed
963
964
   else if (program->stage == sw_gs_copy)
      program->stage |= hw_vs;
965
966
   else if (program->stage == (sw_vs | sw_gs) && gfx9_plus && !ngg)
      program->stage |= hw_gs;
967
968
969
970
   else
      unreachable("Shader stage not implemented");

   program->config = config;
971
972
973
974
   program->info = args->shader_info;
   program->chip_class = args->options->chip_class;
   program->family = args->options->family;
   program->wave_size = args->shader_info->wave_size;
975
   program->lane_mask = program->wave_size == 32 ? s1 : s2;
976

977
978
   program->lds_alloc_granule = args->options->chip_class >= GFX7 ? 512 : 256;
   program->lds_limit = args->options->chip_class >= GFX7 ? 65536 : 32768;
979
   program->vgpr_limit = 256;
980
   program->vgpr_alloc_granule = 3;
981

982
   if (args->options->chip_class >= GFX10) {
983
984
985
      program->physical_sgprs = 2560; /* doesn't matter as long as it's at least 128 * 20 */
      program->sgpr_alloc_granule = 127;
      program->sgpr_limit = 106;
986
      program->vgpr_alloc_granule = program->wave_size == 32 ? 7 : 3;
987
988
989
   } else if (program->chip_class >= GFX8) {
      program->physical_sgprs = 800;
      program->sgpr_alloc_granule = 15;
990
      if (args->options->family == CHIP_TONGA || args->options->family == CHIP_ICELAND)
991
992
         program->sgpr_limit = 94; /* workaround hardware bug */
      else
993
994
995
996
997
         program->sgpr_limit = 102;
   } else {
      program->physical_sgprs = 512;
      program->sgpr_alloc_granule = 7;
      program->sgpr_limit = 104;
998
   }
999

1000
1001
   /* TODO: we don't have to allocate VCC if we don't need it */
   program->needs_vcc = true;
1002

1003
1004
1005
1006
   calc_min_waves(program);
   program->vgpr_limit = get_addr_vgpr_from_waves(program, program->min_waves);
   program->sgpr_limit = get_addr_sgpr_from_waves(program, program->min_waves);

1007
1008
   isel_context ctx = {};
   ctx.program = program;
1009
1010
   ctx.args = args;
   ctx.options = args->options;
1011
1012
   ctx.stage = program->stage;

1013
1014
   get_io_masks(&ctx, shader_count, shaders);

Rhys Perry's avatar
Rhys Perry committed
1015
1016
1017
1018
1019
1020
1021
1022
   unsigned scratch_size = 0;
   if (program->stage == gs_copy_vs) {
      assert(shader_count == 1);
      setup_vs_output_info(&ctx, shaders[0], false, true, &args->shader_info->vs.outinfo);
   } else {
      for (unsigned i = 0; i < shader_count; i++) {
         nir_shader *nir = shaders[i];
         setup_nir(&ctx, nir);
1023

Rhys Perry's avatar
Rhys Perry committed
1024
1025
1026
1027
         if (args->options->dump_preoptir) {
            fprintf(stderr, "NIR shader before instruction selection:\n");
            nir_print_shader(nir, stderr);
         }
1028
      }
1029

Rhys Perry's avatar
Rhys Perry committed
1030
1031
      for (unsigned i = 0; i < shader_count; i++)
         scratch_size = std::max(scratch_size, shaders[i]->scratch_size);
1032
1033
   }

1034
   ctx.program->config->scratch_bytes_per_wave = align(scratch_size * ctx.program->wave_size, 1024);
1035
1036
1037
1038
1039
1040
1041
1042
1043

   ctx.block = ctx.program->create_and_insert_block();
   ctx.block->loop_nest_depth = 0;
   ctx.block->kind = block_kind_top_level;

   return ctx;
}

}