v3d_program.c 29.3 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
/*
 * Copyright © 2014-2017 Broadcom
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include <inttypes.h>
#include "util/u_format.h"
#include "util/u_math.h"
#include "util/u_memory.h"
#include "util/ralloc.h"
#include "util/hash_table.h"
30
#include "util/u_upload_mgr.h"
31
32
33
34
35
36
#include "tgsi/tgsi_dump.h"
#include "tgsi/tgsi_parse.h"
#include "compiler/nir/nir.h"
#include "compiler/nir/nir_builder.h"
#include "nir/tgsi_to_nir.h"
#include "compiler/v3d_compiler.h"
37
#include "v3d_context.h"
38
#include "broadcom/cle/v3d_packet_v33_pack.h"
39
#include "mesa/state_tracker/st_glsl_types.h"
40

41
42
43
44
45
46
static struct v3d_compiled_shader *
v3d_get_compiled_shader(struct v3d_context *v3d, struct v3d_key *key);
static void
v3d_setup_shared_precompile_key(struct v3d_uncompiled_shader *uncompiled,
                                struct v3d_key *key);

47
static gl_varying_slot
48
v3d_get_slot_for_driver_location(nir_shader *s, uint32_t driver_location)
49
50
51
52
53
54
55
56
57
58
{
        nir_foreach_variable(var, &s->outputs) {
                if (var->data.driver_location == driver_location) {
                        return var->data.location;
                }
        }

        return -1;
}

59
60
61
62
63
64
65
66
/**
 * Precomputes the TRANSFORM_FEEDBACK_OUTPUT_DATA_SPEC array for the shader.
 *
 * A shader can have 16 of these specs, and each one of them can write up to
 * 16 dwords.  Since we allow a total of 64 transform feedback output
 * components (not 16 vectors), we have to group the writes of multiple
 * varyings together in a single data spec.
 */
67
static void
68
v3d_set_transform_feedback_outputs(struct v3d_uncompiled_shader *so,
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
                                   const struct pipe_stream_output_info *stream_output)
{
        if (!stream_output->num_outputs)
                return;

        struct v3d_varying_slot slots[PIPE_MAX_SO_OUTPUTS * 4];
        int slot_count = 0;

        for (int buffer = 0; buffer < PIPE_MAX_SO_BUFFERS; buffer++) {
                uint32_t buffer_offset = 0;
                uint32_t vpm_start = slot_count;

                for (int i = 0; i < stream_output->num_outputs; i++) {
                        const struct pipe_stream_output *output =
                                &stream_output->output[i];

                        if (output->output_buffer != buffer)
                                continue;

                        /* We assume that the SO outputs appear in increasing
                         * order in the buffer.
                         */
                        assert(output->dst_offset >= buffer_offset);

                        /* Pad any undefined slots in the output */
                        for (int j = buffer_offset; j < output->dst_offset; j++) {
                                slots[slot_count] =
                                        v3d_slot_from_slot_and_component(VARYING_SLOT_POS, 0);
                                slot_count++;
98
                                buffer_offset++;
99
100
101
102
103
104
                        }

                        /* Set the coordinate shader up to output the
                         * components of this varying.
                         */
                        for (int j = 0; j < output->num_components; j++) {
105
                                gl_varying_slot slot =
106
                                        v3d_get_slot_for_driver_location(so->base.ir.nir, output->register_index);
107

108
                                slots[slot_count] =
109
                                        v3d_slot_from_slot_and_component(slot,
110
111
                                                                         output->start_component + j);
                                slot_count++;
112
                                buffer_offset++;
113
114
115
116
117
118
119
                        }
                }

                uint32_t vpm_size = slot_count - vpm_start;
                if (!vpm_size)
                        continue;

120
121
122
123
124
125
126
127
128
129
130
                uint32_t vpm_start_offset = vpm_start + 6;

                while (vpm_size) {
                        uint32_t write_size = MIN2(vpm_size, 1 << 4);

                        struct V3D33_TRANSFORM_FEEDBACK_OUTPUT_DATA_SPEC unpacked = {
                                /* We need the offset from the coordinate shader's VPM
                                 * output block, which has the [X, Y, Z, W, Xs, Ys]
                                 * values at the start.
                                 */
                                .first_shaded_vertex_value_to_output = vpm_start_offset,
131
                                .number_of_consecutive_vertex_values_to_output_as_32_bit_values = write_size,
132
133
134
                                .output_buffer_to_write_to = buffer,
                        };

135
136
137
138
                        /* GFXH-1559 */
                        assert(unpacked.first_shaded_vertex_value_to_output != 8 ||
                               so->num_tf_specs != 0);

139
140
                        assert(so->num_tf_specs != ARRAY_SIZE(so->tf_specs));
                        V3D33_TRANSFORM_FEEDBACK_OUTPUT_DATA_SPEC_pack(NULL,
141
                                                                       (void *)&so->tf_specs[so->num_tf_specs],
142
                                                                       &unpacked);
143
144
145
146
147
148
149

                        /* If point size is being written by the shader, then
                         * all the VPM start offsets are shifted up by one.
                         * We won't know that until the variant is compiled,
                         * though.
                         */
                        unpacked.first_shaded_vertex_value_to_output++;
150
151
152
153
154

                        /* GFXH-1559 */
                        assert(unpacked.first_shaded_vertex_value_to_output != 8 ||
                               so->num_tf_specs != 0);

155
156
157
158
                        V3D33_TRANSFORM_FEEDBACK_OUTPUT_DATA_SPEC_pack(NULL,
                                                                       (void *)&so->tf_specs_psiz[so->num_tf_specs],
                                                                       &unpacked);
                        so->num_tf_specs++;
159
160
161
                        vpm_start_offset += write_size;
                        vpm_size -= write_size;
                }
162
163
                so->base.stream_output.stride[buffer] =
                        stream_output->stride[buffer];
164
165
166
167
168
169
170
171
172
173
174
175
176
177
        }

        so->num_tf_outputs = slot_count;
        so->tf_outputs = ralloc_array(so->base.ir.nir, struct v3d_varying_slot,
                                      slot_count);
        memcpy(so->tf_outputs, slots, sizeof(*slots) * slot_count);
}

static int
type_size(const struct glsl_type *type)
{
        return glsl_count_attribute_slots(type, false);
}

178
179
180
181
182
183
static int
uniforms_type_size(const struct glsl_type *type)
{
        return st_glsl_storage_type_size(type, false);
}

184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
/**
 * Precompiles a shader variant at shader state creation time if
 * V3D_DEBUG=precompile is set.  Used for shader-db
 * (https://gitlab.freedesktop.org/mesa/shader-db)
 */
static void
v3d_shader_precompile(struct v3d_context *v3d,
                      struct v3d_uncompiled_shader *so)
{
        nir_shader *s = so->base.ir.nir;

        if (s->info.stage == MESA_SHADER_FRAGMENT) {
                struct v3d_fs_key key = {
                        .base.shader_state = so,
                };

200
201
202
                nir_foreach_variable(var, &s->outputs) {
                        if (var->data.location == FRAG_RESULT_COLOR) {
                                key.nr_cbufs = 1;
203
                        } else if (var->data.location >= FRAG_RESULT_DATA0) {
204
205
206
207
208
209
                                key.nr_cbufs = MAX2(key.nr_cbufs,
                                                    var->data.location -
                                                    FRAG_RESULT_DATA0 + 1);
                        }
                }

210
211
212
213
214
215
216
217
218
219
                v3d_setup_shared_precompile_key(so, &key.base);
                v3d_get_compiled_shader(v3d, &key.base);
        } else {
                struct v3d_vs_key key = {
                        .base.shader_state = so,
                };

                v3d_setup_shared_precompile_key(so, &key.base);

                /* Compile VS: All outputs */
220
221
222
223
224
225
226
227
                nir_foreach_variable(var, &s->outputs) {
                        unsigned array_len = MAX2(glsl_get_length(var->type), 1);
                        assert(array_len == 1);
                        (void)array_len;

                        int slot = var->data.location;
                        for (int i = 0; i < glsl_get_components(var->type); i++) {
                                int swiz = var->data.location_frac + i;
228
                                key.fs_inputs[key.num_fs_inputs++] =
229
230
                                        v3d_slot_from_slot_and_component(slot,
                                                                         swiz);
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
                        }
                }

                v3d_get_compiled_shader(v3d, &key.base);

                /* Compile VS bin shader: only position (XXX: include TF) */
                key.is_coord = true;
                key.num_fs_inputs = 0;
                for (int i = 0; i < 4; i++) {
                        key.fs_inputs[key.num_fs_inputs++] =
                                v3d_slot_from_slot_and_component(VARYING_SLOT_POS,
                                                                 i);
                }
                v3d_get_compiled_shader(v3d, &key.base);
        }
}

248
static void *
249
v3d_shader_state_create(struct pipe_context *pctx,
250
251
                        const struct pipe_shader_state *cso)
{
252
253
        struct v3d_context *v3d = v3d_context(pctx);
        struct v3d_uncompiled_shader *so = CALLOC_STRUCT(v3d_uncompiled_shader);
254
255
256
        if (!so)
                return NULL;

257
        so->program_id = v3d->next_uncompiled_program_id++;
258
259
260
261
262
263
264
265
266

        nir_shader *s;

        if (cso->type == PIPE_SHADER_IR_NIR) {
                /* The backend takes ownership of the NIR shader on state
                 * creation.
                 */
                s = cso->ir.nir;

267
268
                NIR_PASS_V(s, nir_lower_io, nir_var_uniform,
                           uniforms_type_size,
269
270
271
272
273
274
275
276
277
278
279
280
281
                           (nir_lower_io_options)0);
        } else {
                assert(cso->type == PIPE_SHADER_IR_TGSI);

                if (V3D_DEBUG & V3D_DEBUG_TGSI) {
                        fprintf(stderr, "prog %d TGSI:\n",
                                so->program_id);
                        tgsi_dump(cso->tokens, 0);
                        fprintf(stderr, "\n");
                }
                s = tgsi_to_nir(cso->tokens, &v3d_nir_options);
        }

282
283
        nir_variable_mode lower_mode = nir_var_all & ~nir_var_uniform;
        if (s->info.stage == MESA_SHADER_VERTEX)
284
                lower_mode &= ~(nir_var_shader_in | nir_var_shader_out);
285
        NIR_PASS_V(s, nir_lower_io, lower_mode,
286
287
288
                   type_size,
                   (nir_lower_io_options)0);

289
290
291
292
293
294
295
296
        NIR_PASS_V(s, nir_opt_global_to_local);
        NIR_PASS_V(s, nir_lower_regs_to_ssa);
        NIR_PASS_V(s, nir_normalize_cubemap_coords);

        NIR_PASS_V(s, nir_lower_load_const_to_scalar);

        v3d_optimize_nir(s);

297
        NIR_PASS_V(s, nir_remove_dead_variables, nir_var_function_temp);
298
299
300
301
302
303
304

        /* Garbage collect dead instructions */
        nir_sweep(s);

        so->base.type = PIPE_SHADER_IR_NIR;
        so->base.ir.nir = s;

305
        v3d_set_transform_feedback_outputs(so, &cso->stream_output);
306
307

        if (V3D_DEBUG & (V3D_DEBUG_NIR |
308
                         v3d_debug_flag_for_shader_stage(s->info.stage))) {
309
                fprintf(stderr, "%s prog %d NIR:\n",
310
                        gl_shader_stage_name(s->info.stage),
311
312
313
314
315
                        so->program_id);
                nir_print_shader(s, stderr);
                fprintf(stderr, "\n");
        }

316
317
318
        if (V3D_DEBUG & V3D_DEBUG_PRECOMPILE)
                v3d_shader_precompile(v3d, so);

319
320
321
        return so;
}

322
323
324
325
326
327
328
329
static void
v3d_shader_debug_output(const char *message, void *data)
{
        struct v3d_context *v3d = data;

        pipe_debug_message(&v3d->debug, SHADER_INFO, "%s", message);
}

330
331
static struct v3d_compiled_shader *
v3d_get_compiled_shader(struct v3d_context *v3d, struct v3d_key *key)
332
{
333
        struct v3d_uncompiled_shader *shader_state = key->shader_state;
334
335
336
337
        nir_shader *s = shader_state->base.ir.nir;

        struct hash_table *ht;
        uint32_t key_size;
338
        if (s->info.stage == MESA_SHADER_FRAGMENT) {
339
                ht = v3d->fs_cache;
340
341
                key_size = sizeof(struct v3d_fs_key);
        } else {
342
                ht = v3d->vs_cache;
343
344
345
346
347
348
349
                key_size = sizeof(struct v3d_vs_key);
        }

        struct hash_entry *entry = _mesa_hash_table_search(ht, key);
        if (entry)
                return entry->data;

350
351
        struct v3d_compiled_shader *shader =
                rzalloc(NULL, struct v3d_compiled_shader);
352
353
354
355
356
357
358

        int program_id = shader_state->program_id;
        int variant_id =
                p_atomic_inc_return(&shader_state->compiled_variant_count);
        uint64_t *qpu_insts;
        uint32_t shader_size;

359
360
361
362
363
364
        qpu_insts = v3d_compile(v3d->screen->compiler, key,
                                &shader->prog_data.base, s,
                                v3d_shader_debug_output,
                                v3d,
                                program_id, variant_id, &shader_size);
        ralloc_steal(shader, shader->prog_data.base);
365

366
        v3d_set_shader_uniform_dirty_flags(shader);
367

368
369
370
371
        if (shader_size) {
                u_upload_data(v3d->state_uploader, 0, shader_size, 8,
                              qpu_insts, &shader->offset, &shader->resource);
        }
372
373
374

        free(qpu_insts);

375
        struct v3d_key *dup_key;
376
377
378
379
        dup_key = ralloc_size(shader, key_size);
        memcpy(dup_key, key, key_size);
        _mesa_hash_table_insert(ht, dup_key, shader);

380
        if (shader->prog_data.base->spill_size >
381
            v3d->prog.spill_size_per_thread) {
382
383
384
385
386
387
                /* Max 4 QPUs per slice, 3 slices per core. We only do single
                 * core so far.  This overallocates memory on smaller cores.
                 */
                int total_spill_size =
                        4 * 3 * shader->prog_data.base->spill_size;

388
389
                v3d_bo_unreference(&v3d->prog.spill_bo);
                v3d->prog.spill_bo = v3d_bo_alloc(v3d->screen,
390
                                                  total_spill_size, "spill");
391
                v3d->prog.spill_size_per_thread =
392
393
394
                        shader->prog_data.base->spill_size;
        }

395
396
397
        return shader;
}

398
399
400
401
402
403
404
static void
v3d_free_compiled_shader(struct v3d_compiled_shader *shader)
{
        pipe_resource_reference(&shader->resource, NULL);
        ralloc_free(shader);
}

405
static void
406
407
v3d_setup_shared_key(struct v3d_context *v3d, struct v3d_key *key,
                     struct v3d_texture_stateobj *texstate)
408
{
409
        const struct v3d_device_info *devinfo = &v3d->screen->devinfo;
410

411
412
        for (int i = 0; i < texstate->num_textures; i++) {
                struct pipe_sampler_view *sampler = texstate->textures[i];
413
                struct v3d_sampler_view *v3d_sampler = v3d_sampler_view(sampler);
414
415
416
417
418
419
420
                struct pipe_sampler_state *sampler_state =
                        texstate->samplers[i];

                if (!sampler)
                        continue;

                key->tex[i].return_size =
421
                        v3d_get_tex_return_size(devinfo,
422
                                                sampler->format,
423
                                                sampler_state->compare_mode);
424
425
426
427
428
429
430

                /* For 16-bit, we set up the sampler to always return 2
                 * channels (meaning no recompiles for most statechanges),
                 * while for 32 we actually scale the returns with channels.
                 */
                if (key->tex[i].return_size == 16) {
                        key->tex[i].return_channels = 2;
431
432
                } else if (devinfo->ver > 40) {
                        key->tex[i].return_channels = 4;
433
434
                } else {
                        key->tex[i].return_channels =
435
                                v3d_get_tex_return_channels(devinfo,
436
                                                            sampler->format);
437
438
                }

439
                if (key->tex[i].return_size == 32 && devinfo->ver < 40) {
440
                        memcpy(key->tex[i].swizzle,
441
442
                               v3d_sampler->swizzle,
                               sizeof(v3d_sampler->swizzle));
443
444
445
446
447
448
449
450
451
452
                } else {
                        /* For 16-bit returns, we let the sampler state handle
                         * the swizzle.
                         */
                        key->tex[i].swizzle[0] = PIPE_SWIZZLE_X;
                        key->tex[i].swizzle[1] = PIPE_SWIZZLE_Y;
                        key->tex[i].swizzle[2] = PIPE_SWIZZLE_Z;
                        key->tex[i].swizzle[3] = PIPE_SWIZZLE_W;
                }

453
                if (sampler) {
454
455
456
457
458
459
                        key->tex[i].clamp_s =
                                sampler_state->wrap_s == PIPE_TEX_WRAP_CLAMP;
                        key->tex[i].clamp_t =
                                sampler_state->wrap_t == PIPE_TEX_WRAP_CLAMP;
                        key->tex[i].clamp_r =
                                sampler_state->wrap_r == PIPE_TEX_WRAP_CLAMP;
460
461
462
                }
        }

463
        key->ucp_enables = v3d->rasterizer->base.clip_plane_enable;
464
465
}

466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
static void
v3d_setup_shared_precompile_key(struct v3d_uncompiled_shader *uncompiled,
                                struct v3d_key *key)
{
        nir_shader *s = uncompiled->base.ir.nir;

        for (int i = 0; i < s->info.num_textures; i++) {
                key->tex[i].return_size = 16;
                key->tex[i].return_channels = 2;

                key->tex[i].swizzle[0] = PIPE_SWIZZLE_X;
                key->tex[i].swizzle[1] = PIPE_SWIZZLE_Y;
                key->tex[i].swizzle[2] = PIPE_SWIZZLE_Z;
                key->tex[i].swizzle[3] = PIPE_SWIZZLE_W;
        }
}

483
static void
484
v3d_update_compiled_fs(struct v3d_context *v3d, uint8_t prim_mode)
485
{
486
        struct v3d_job *job = v3d->job;
487
488
        struct v3d_fs_key local_key;
        struct v3d_fs_key *key = &local_key;
489
        nir_shader *s = v3d->prog.bind_fs->base.ir.nir;
490

491
        if (!(v3d->dirty & (VC5_DIRTY_PRIM_MODE |
492
493
494
495
                            VC5_DIRTY_BLEND |
                            VC5_DIRTY_FRAMEBUFFER |
                            VC5_DIRTY_ZSA |
                            VC5_DIRTY_RASTERIZER |
496
                            VC5_DIRTY_SAMPLE_STATE |
497
498
499
500
501
502
                            VC5_DIRTY_FRAGTEX |
                            VC5_DIRTY_UNCOMPILED_FS))) {
                return;
        }

        memset(key, 0, sizeof(*key));
503
        v3d_setup_shared_key(v3d, &key->base, &v3d->tex[PIPE_SHADER_FRAGMENT]);
504
        key->base.shader_state = v3d->prog.bind_fs;
505
506
507
        key->is_points = (prim_mode == PIPE_PRIM_POINTS);
        key->is_lines = (prim_mode >= PIPE_PRIM_LINES &&
                         prim_mode <= PIPE_PRIM_LINE_STRIP);
508
        key->clamp_color = v3d->rasterizer->base.clamp_fragment_color;
509
510
        if (v3d->blend->base.logicop_enable) {
                key->logicop_func = v3d->blend->base.logicop_func;
511
512
513
514
        } else {
                key->logicop_func = PIPE_LOGICOP_COPY;
        }
        if (job->msaa) {
515
516
                key->msaa = v3d->rasterizer->base.multisample;
                key->sample_coverage = (v3d->rasterizer->base.multisample &&
517
                                        v3d->sample_mask != (1 << V3D_MAX_SAMPLES) - 1);
518
519
                key->sample_alpha_to_coverage = v3d->blend->base.alpha_to_coverage;
                key->sample_alpha_to_one = v3d->blend->base.alpha_to_one;
520
521
        }

522
523
524
        key->depth_enabled = (v3d->zsa->base.depth.enabled ||
                              v3d->zsa->base.stencil[0].enabled);
        if (v3d->zsa->base.alpha.enabled) {
525
                key->alpha_test = true;
526
                key->alpha_test_func = v3d->zsa->base.alpha.func;
527
528
        }

529
530
531
        /* gl_FragColor's propagation to however many bound color buffers
         * there are means that the buffer count needs to be in the key.
         */
532
533
        key->nr_cbufs = v3d->framebuffer.nr_cbufs;
        key->swap_color_rb = v3d->swap_color_rb;
534
535

        for (int i = 0; i < key->nr_cbufs; i++) {
536
                struct pipe_surface *cbuf = v3d->framebuffer.cbufs[i];
537
538
539
                if (!cbuf)
                        continue;

540
541
542
                const struct util_format_description *desc =
                        util_format_description(cbuf->format);

543
544
545
546
                if (desc->channel[0].type == UTIL_FORMAT_TYPE_FLOAT &&
                    desc->channel[0].size == 32) {
                        key->f32_color_rb |= 1 << i;
                }
547

548
                if (s->info.fs.untyped_color_outputs) {
549
550
551
552
553
                        if (util_format_is_pure_uint(cbuf->format))
                                key->uint_color_rb |= 1 << i;
                        else if (util_format_is_pure_sint(cbuf->format))
                                key->int_color_rb |= 1 << i;
                }
554
555
556
557
        }

        if (key->is_points) {
                key->point_sprite_mask =
558
                        v3d->rasterizer->base.sprite_coord_enable;
559
                key->point_coord_upper_left =
560
                        (v3d->rasterizer->base.sprite_coord_mode ==
561
562
563
                         PIPE_SPRITE_COORD_UPPER_LEFT);
        }

564
565
        key->light_twoside = v3d->rasterizer->base.light_twoside;
        key->shade_model_flat = v3d->rasterizer->base.flatshade;
566

567
568
569
        struct v3d_compiled_shader *old_fs = v3d->prog.fs;
        v3d->prog.fs = v3d_get_compiled_shader(v3d, &key->base);
        if (v3d->prog.fs == old_fs)
570
571
                return;

572
        v3d->dirty |= VC5_DIRTY_COMPILED_FS;
573

574
        if (old_fs) {
575
                if (v3d->prog.fs->prog_data.fs->flat_shade_flags !=
576
                    old_fs->prog_data.fs->flat_shade_flags) {
577
                        v3d->dirty |= VC5_DIRTY_FLAT_SHADE_FLAGS;
578
579
                }

580
581
582
583
584
                if (v3d->prog.fs->prog_data.fs->noperspective_flags !=
                    old_fs->prog_data.fs->noperspective_flags) {
                        v3d->dirty |= VC5_DIRTY_NOPERSPECTIVE_FLAGS;
                }

585
                if (v3d->prog.fs->prog_data.fs->centroid_flags !=
586
                    old_fs->prog_data.fs->centroid_flags) {
587
                        v3d->dirty |= VC5_DIRTY_CENTROID_FLAGS;
588
                }
589
590
        }

591
        if (old_fs && memcmp(v3d->prog.fs->prog_data.fs->input_slots,
592
                             old_fs->prog_data.fs->input_slots,
593
594
                             sizeof(v3d->prog.fs->prog_data.fs->input_slots))) {
                v3d->dirty |= VC5_DIRTY_FS_INPUTS;
595
596
597
598
        }
}

static void
599
v3d_update_compiled_vs(struct v3d_context *v3d, uint8_t prim_mode)
600
601
602
603
{
        struct v3d_vs_key local_key;
        struct v3d_vs_key *key = &local_key;

604
        if (!(v3d->dirty & (VC5_DIRTY_PRIM_MODE |
605
606
607
608
609
610
611
612
613
                            VC5_DIRTY_RASTERIZER |
                            VC5_DIRTY_VERTTEX |
                            VC5_DIRTY_VTXSTATE |
                            VC5_DIRTY_UNCOMPILED_VS |
                            VC5_DIRTY_FS_INPUTS))) {
                return;
        }

        memset(key, 0, sizeof(*key));
614
        v3d_setup_shared_key(v3d, &key->base, &v3d->tex[PIPE_SHADER_VERTEX]);
615
616
        key->base.shader_state = v3d->prog.bind_vs;
        key->num_fs_inputs = v3d->prog.fs->prog_data.fs->base.num_inputs;
617
        STATIC_ASSERT(sizeof(key->fs_inputs) ==
618
619
                      sizeof(v3d->prog.fs->prog_data.fs->input_slots));
        memcpy(key->fs_inputs, v3d->prog.fs->prog_data.fs->input_slots,
620
               sizeof(key->fs_inputs));
621
        key->clamp_color = v3d->rasterizer->base.clamp_vertex_color;
622
623
624

        key->per_vertex_point_size =
                (prim_mode == PIPE_PRIM_POINTS &&
625
                 v3d->rasterizer->base.point_size_per_vertex);
626

627
628
629
630
631
        struct v3d_compiled_shader *vs =
                v3d_get_compiled_shader(v3d, &key->base);
        if (vs != v3d->prog.vs) {
                v3d->prog.vs = vs;
                v3d->dirty |= VC5_DIRTY_COMPILED_VS;
632
633
634
635
        }

        key->is_coord = true;
        /* Coord shaders only output varyings used by transform feedback. */
636
        struct v3d_uncompiled_shader *shader_state = key->base.shader_state;
637
638
639
640
641
642
643
644
645
646
        memcpy(key->fs_inputs, shader_state->tf_outputs,
               sizeof(*key->fs_inputs) * shader_state->num_tf_outputs);
        if (shader_state->num_tf_outputs < key->num_fs_inputs) {
                memset(&key->fs_inputs[shader_state->num_tf_outputs],
                       0,
                       sizeof(*key->fs_inputs) * (key->num_fs_inputs -
                                                  shader_state->num_tf_outputs));
        }
        key->num_fs_inputs = shader_state->num_tf_outputs;

647
648
649
650
651
        struct v3d_compiled_shader *cs =
                v3d_get_compiled_shader(v3d, &key->base);
        if (cs != v3d->prog.cs) {
                v3d->prog.cs = cs;
                v3d->dirty |= VC5_DIRTY_COMPILED_CS;
652
653
654
655
        }
}

void
656
v3d_update_compiled_shaders(struct v3d_context *v3d, uint8_t prim_mode)
657
{
658
659
        v3d_update_compiled_fs(v3d, prim_mode);
        v3d_update_compiled_vs(v3d, prim_mode);
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
}

static uint32_t
fs_cache_hash(const void *key)
{
        return _mesa_hash_data(key, sizeof(struct v3d_fs_key));
}

static uint32_t
vs_cache_hash(const void *key)
{
        return _mesa_hash_data(key, sizeof(struct v3d_vs_key));
}

static bool
fs_cache_compare(const void *key1, const void *key2)
{
        return memcmp(key1, key2, sizeof(struct v3d_fs_key)) == 0;
}

static bool
vs_cache_compare(const void *key1, const void *key2)
{
        return memcmp(key1, key2, sizeof(struct v3d_vs_key)) == 0;
}

static void
delete_from_cache_if_matches(struct hash_table *ht,
688
                             struct v3d_compiled_shader **last_compile,
689
                             struct hash_entry *entry,
690
                             struct v3d_uncompiled_shader *so)
691
692
693
694
{
        const struct v3d_key *key = entry->key;

        if (key->shader_state == so) {
695
                struct v3d_compiled_shader *shader = entry->data;
696
697
698
699
700
                _mesa_hash_table_remove(ht, entry);

                if (shader == *last_compile)
                        *last_compile = NULL;

701
                v3d_free_compiled_shader(shader);
702
703
704
705
        }
}

static void
706
v3d_shader_state_delete(struct pipe_context *pctx, void *hwcso)
707
{
708
709
        struct v3d_context *v3d = v3d_context(pctx);
        struct v3d_uncompiled_shader *so = hwcso;
710

711
712
        hash_table_foreach(v3d->fs_cache, entry) {
                delete_from_cache_if_matches(v3d->fs_cache, &v3d->prog.fs,
713
714
                                             entry, so);
        }
715
716
        hash_table_foreach(v3d->vs_cache, entry) {
                delete_from_cache_if_matches(v3d->vs_cache, &v3d->prog.vs,
717
718
719
720
721
722
723
724
                                             entry, so);
        }

        ralloc_free(so->base.ir.nir);
        free(so);
}

static void
725
v3d_fp_state_bind(struct pipe_context *pctx, void *hwcso)
726
{
727
728
729
        struct v3d_context *v3d = v3d_context(pctx);
        v3d->prog.bind_fs = hwcso;
        v3d->dirty |= VC5_DIRTY_UNCOMPILED_FS;
730
731
732
}

static void
733
v3d_vp_state_bind(struct pipe_context *pctx, void *hwcso)
734
{
735
736
737
        struct v3d_context *v3d = v3d_context(pctx);
        v3d->prog.bind_vs = hwcso;
        v3d->dirty |= VC5_DIRTY_UNCOMPILED_VS;
738
739
740
}

void
741
v3d_program_init(struct pipe_context *pctx)
742
{
743
        struct v3d_context *v3d = v3d_context(pctx);
744

745
746
        pctx->create_vs_state = v3d_shader_state_create;
        pctx->delete_vs_state = v3d_shader_state_delete;
747

748
749
        pctx->create_fs_state = v3d_shader_state_create;
        pctx->delete_fs_state = v3d_shader_state_delete;
750

751
752
        pctx->bind_fs_state = v3d_fp_state_bind;
        pctx->bind_vs_state = v3d_vp_state_bind;
753

754
        v3d->fs_cache = _mesa_hash_table_create(pctx, fs_cache_hash,
755
                                                fs_cache_compare);
756
        v3d->vs_cache = _mesa_hash_table_create(pctx, vs_cache_hash,
757
758
759
760
                                                vs_cache_compare);
}

void
761
v3d_program_fini(struct pipe_context *pctx)
762
{
763
        struct v3d_context *v3d = v3d_context(pctx);
764

765
766
        hash_table_foreach(v3d->fs_cache, entry) {
                struct v3d_compiled_shader *shader = entry->data;
767
                v3d_free_compiled_shader(shader);
768
                _mesa_hash_table_remove(v3d->fs_cache, entry);
769
770
        }

771
772
        hash_table_foreach(v3d->vs_cache, entry) {
                struct v3d_compiled_shader *shader = entry->data;
773
                v3d_free_compiled_shader(shader);
774
                _mesa_hash_table_remove(v3d->vs_cache, entry);
775
        }
776
777

        v3d_bo_unreference(&v3d->prog.spill_bo);
778
}