brw_fs.cpp 237 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
/*
 * Copyright © 2010 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
22 23 24
 */

/** @file brw_fs.cpp
25
 *
26 27 28
 * This file drives the GLSL IR -> LIR translation, contains the
 * optimizations on the LIR, and drives the generation of native code
 * from the LIR.
29 30 31 32
 */

#include "main/macros.h"
#include "brw_eu.h"
33
#include "brw_fs.h"
34
#include "brw_nir.h"
35
#include "brw_vec4_gs_visitor.h"
36
#include "brw_cfg.h"
37
#include "brw_dead_control_flow.h"
38
#include "common/gen_debug.h"
39
#include "compiler/glsl_types.h"
40
#include "compiler/nir/nir_builder.h"
41
#include "program/prog_parameter.h"
42

43 44
using namespace brw;

45
static unsigned get_lowered_simd_width(const struct gen_device_info *devinfo,
46 47
                                       const fs_inst *inst);

48
void
49
fs_inst::init(enum opcode opcode, uint8_t exec_size, const fs_reg &dst,
50
              const fs_reg *src, unsigned sources)
51 52
{
   memset(this, 0, sizeof(*this));
53

54 55 56 57
   this->src = new fs_reg[MAX2(sources, 3)];
   for (unsigned i = 0; i < sources; i++)
      this->src[i] = src[i];

58 59
   this->opcode = opcode;
   this->dst = dst;
60
   this->sources = sources;
61
   this->exec_size = exec_size;
62
   this->base_mrf = -1;
63 64 65 66

   assert(dst.file != IMM && dst.file != UNIFORM);

   assert(this->exec_size != 0);
67

68 69
   this->conditional_mod = BRW_CONDITIONAL_NONE;

70
   /* This will be the case for almost all instructions. */
71
   switch (dst.file) {
Matt Turner's avatar
Matt Turner committed
72
   case VGRF:
73 74
   case ARF:
   case FIXED_GRF:
75
   case MRF:
76
   case ATTR:
77
      this->size_written = dst.component_size(exec_size);
78 79
      break;
   case BAD_FILE:
80
      this->size_written = 0;
81 82 83 84 85
      break;
   case IMM:
   case UNIFORM:
      unreachable("Invalid destination register file");
   }
86 87

   this->writes_accumulator = false;
88 89
}

90 91
fs_inst::fs_inst()
{
92
   init(BRW_OPCODE_NOP, 8, dst, NULL, 0);
93 94 95 96
}

fs_inst::fs_inst(enum opcode opcode, uint8_t exec_size)
{
97
   init(opcode, exec_size, reg_undef, NULL, 0);
98 99
}

100
fs_inst::fs_inst(enum opcode opcode, uint8_t exec_size, const fs_reg &dst)
101
{
102
   init(opcode, exec_size, dst, NULL, 0);
103 104 105 106 107
}

fs_inst::fs_inst(enum opcode opcode, uint8_t exec_size, const fs_reg &dst,
                 const fs_reg &src0)
{
108
   const fs_reg src[1] = { src0 };
109
   init(opcode, exec_size, dst, src, 1);
110 111
}

112 113 114
fs_inst::fs_inst(enum opcode opcode, uint8_t exec_size, const fs_reg &dst,
                 const fs_reg &src0, const fs_reg &src1)
{
115
   const fs_reg src[2] = { src0, src1 };
116
   init(opcode, exec_size, dst, src, 2);
117 118
}

119 120 121
fs_inst::fs_inst(enum opcode opcode, uint8_t exec_size, const fs_reg &dst,
                 const fs_reg &src0, const fs_reg &src1, const fs_reg &src2)
{
122
   const fs_reg src[3] = { src0, src1, src2 };
123
   init(opcode, exec_size, dst, src, 3);
124 125
}

126
fs_inst::fs_inst(enum opcode opcode, uint8_t exec_width, const fs_reg &dst,
127
                 const fs_reg src[], unsigned sources)
128 129
{
   init(opcode, exec_width, dst, src, sources);
130 131
}

132 133 134
fs_inst::fs_inst(const fs_inst &that)
{
   memcpy(this, &that, sizeof(that));
135

136
   this->src = new fs_reg[MAX2(that.sources, 3)];
137

138
   for (unsigned i = 0; i < that.sources; i++)
139
      this->src[i] = that.src[i];
140 141
}

142 143 144 145 146
fs_inst::~fs_inst()
{
   delete[] this->src;
}

147 148 149 150
void
fs_inst::resize_sources(uint8_t num_sources)
{
   if (this->sources != num_sources) {
151 152 153 154 155 156 157
      fs_reg *src = new fs_reg[MAX2(num_sources, 3)];

      for (unsigned i = 0; i < MIN2(this->sources, num_sources); ++i)
         src[i] = this->src[i];

      delete[] this->src;
      this->src = src;
158 159 160 161
      this->sources = num_sources;
   }
}

162 163 164
void
fs_visitor::VARYING_PULL_CONSTANT_LOAD(const fs_builder &bld,
                                       const fs_reg &dst,
165 166
                                       const fs_reg &surf_index,
                                       const fs_reg &varying_offset,
167
                                       uint32_t const_offset)
168
{
169 170 171 172
   /* We have our constant surface use a pitch of 4 bytes, so our index can
    * be any component of a vector, and then we load 4 contiguous
    * components starting from that.
    *
173 174 175 176 177 178
    * We break down the const_offset to a portion added to the variable offset
    * and a portion done using fs_reg::offset, which means that if you have
    * GLSL using something like "uniform vec4 a[20]; gl_FragColor = a[i]",
    * we'll temporarily generate 4 vec4 loads from offset i * 4, and CSE can
    * later notice that those loads are all the same and eliminate the
    * redundant ones.
179
    */
180
   fs_reg vec4_offset = vgrf(glsl_type::uint_type);
181
   bld.ADD(vec4_offset, varying_offset, brw_imm_ud(const_offset & ~0xf));
182

183 184 185 186 187 188
   /* The pull load message will load a vec4 (16 bytes). If we are loading
    * a double this means we are only loading 2 elements worth of data.
    * We also want to use a 32-bit data type for the dst of the load operation
    * so other parts of the driver don't get confused about the size of the
    * result.
    */
189
   fs_reg vec4_result = bld.vgrf(BRW_REGISTER_TYPE_F, 4);
190 191
   fs_inst *inst = bld.emit(FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_LOGICAL,
                            vec4_result, surf_index, vec4_offset);
192
   inst->size_written = 4 * vec4_result.component_size(inst->exec_size);
193

194 195 196
   fs_reg dw = offset(vec4_result, bld, (const_offset & 0xf) / 4);
   switch (type_sz(dst.type)) {
   case 2:
197
      shuffle_32bit_load_result_to_16bit_data(bld, dst, dw, 0, 1);
198 199 200 201 202 203 204 205 206 207
      bld.MOV(dst, subscript(dw, dst.type, (const_offset / 2) & 1));
      break;
   case 4:
      bld.MOV(dst, retype(dw, dst.type));
      break;
   case 8:
      shuffle_32bit_load_result_to_64bit_data(bld, dst, dw, 1);
      break;
   default:
      unreachable("Unsupported bit_size");
208
   }
209 210
}

211 212 213 214
/**
 * A helper for MOV generation for fixing up broken hardware SEND dependency
 * handling.
 */
215 216
void
fs_visitor::DEP_RESOLVE_MOV(const fs_builder &bld, int grf)
217 218 219 220
{
   /* The caller always wants uncompressed to emit the minimal extra
    * dependencies, and to avoid having to deal with aligning its regs to 2.
    */
221 222
   const fs_builder ubld = bld.annotate("send dependency resolve")
                              .half(0);
223

Matt Turner's avatar
Matt Turner committed
224
   ubld.MOV(ubld.null_reg_f(), fs_reg(VGRF, grf, BRW_REGISTER_TYPE_F));
225 226
}

227
bool
228
fs_inst::equals(fs_inst *inst) const
229 230 231 232 233 234 235
{
   return (opcode == inst->opcode &&
           dst.equals(inst->dst) &&
           src[0].equals(inst->src[0]) &&
           src[1].equals(inst->src[1]) &&
           src[2].equals(inst->src[2]) &&
           saturate == inst->saturate &&
236
           predicate == inst->predicate &&
237 238 239 240 241
           conditional_mod == inst->conditional_mod &&
           mlen == inst->mlen &&
           base_mrf == inst->base_mrf &&
           target == inst->target &&
           eot == inst->eot &&
242
           header_size == inst->header_size &&
243
           shadow_compare == inst->shadow_compare &&
244
           exec_size == inst->exec_size &&
245 246 247
           offset == inst->offset);
}

248
bool
249
fs_inst::is_send_from_grf() const
250
{
251 252 253 254 255 256
   switch (opcode) {
   case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_GEN7:
   case SHADER_OPCODE_SHADER_TIME_ADD:
   case FS_OPCODE_INTERPOLATE_AT_SAMPLE:
   case FS_OPCODE_INTERPOLATE_AT_SHARED_OFFSET:
   case FS_OPCODE_INTERPOLATE_AT_PER_SLOT_OFFSET:
257
   case SHADER_OPCODE_UNTYPED_ATOMIC:
258
   case SHADER_OPCODE_UNTYPED_SURFACE_READ:
259
   case SHADER_OPCODE_UNTYPED_SURFACE_WRITE:
260
   case SHADER_OPCODE_BYTE_SCATTERED_WRITE:
261
   case SHADER_OPCODE_BYTE_SCATTERED_READ:
262 263 264
   case SHADER_OPCODE_TYPED_ATOMIC:
   case SHADER_OPCODE_TYPED_SURFACE_READ:
   case SHADER_OPCODE_TYPED_SURFACE_WRITE:
265
   case SHADER_OPCODE_URB_WRITE_SIMD8:
266 267 268
   case SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT:
   case SHADER_OPCODE_URB_WRITE_SIMD8_MASKED:
   case SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT:
269
   case SHADER_OPCODE_URB_READ_SIMD8:
270
   case SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT:
271 272
      return true;
   case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD:
Matt Turner's avatar
Matt Turner committed
273
      return src[1].file == VGRF;
274
   case FS_OPCODE_FB_WRITE:
275
   case FS_OPCODE_FB_READ:
Matt Turner's avatar
Matt Turner committed
276
      return src[0].file == VGRF;
277 278
   default:
      if (is_tex())
Matt Turner's avatar
Matt Turner committed
279
         return src[0].file == VGRF;
280 281 282

      return false;
   }
283 284
}

285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349
/**
 * Returns true if this instruction's sources and destinations cannot
 * safely be the same register.
 *
 * In most cases, a register can be written over safely by the same
 * instruction that is its last use.  For a single instruction, the
 * sources are dereferenced before writing of the destination starts
 * (naturally).
 *
 * However, there are a few cases where this can be problematic:
 *
 * - Virtual opcodes that translate to multiple instructions in the
 *   code generator: if src == dst and one instruction writes the
 *   destination before a later instruction reads the source, then
 *   src will have been clobbered.
 *
 * - SIMD16 compressed instructions with certain regioning (see below).
 *
 * The register allocator uses this information to set up conflicts between
 * GRF sources and the destination.
 */
bool
fs_inst::has_source_and_destination_hazard() const
{
   switch (opcode) {
   case FS_OPCODE_PACK_HALF_2x16_SPLIT:
      /* Multiple partial writes to the destination */
      return true;
   default:
      /* The SIMD16 compressed instruction
       *
       * add(16)      g4<1>F      g4<8,8,1>F   g6<8,8,1>F
       *
       * is actually decoded in hardware as:
       *
       * add(8)       g4<1>F      g4<8,8,1>F   g6<8,8,1>F
       * add(8)       g5<1>F      g5<8,8,1>F   g7<8,8,1>F
       *
       * Which is safe.  However, if we have uniform accesses
       * happening, we get into trouble:
       *
       * add(8)       g4<1>F      g4<0,1,0>F   g6<8,8,1>F
       * add(8)       g5<1>F      g4<0,1,0>F   g7<8,8,1>F
       *
       * Now our destination for the first instruction overwrote the
       * second instruction's src0, and we get garbage for those 8
       * pixels.  There's a similar issue for the pre-gen6
       * pixel_x/pixel_y, which are registers of 16-bit values and thus
       * would get stomped by the first decode as well.
       */
      if (exec_size == 16) {
         for (int i = 0; i < sources; i++) {
            if (src[i].file == VGRF && (src[i].stride == 0 ||
                                        src[i].type == BRW_REGISTER_TYPE_UW ||
                                        src[i].type == BRW_REGISTER_TYPE_W ||
                                        src[i].type == BRW_REGISTER_TYPE_UB ||
                                        src[i].type == BRW_REGISTER_TYPE_B)) {
               return true;
            }
         }
      }
      return false;
   }
}

350 351 352 353 354 355 356
bool
fs_inst::is_copy_payload(const brw::simple_allocator &grf_alloc) const
{
   if (this->opcode != SHADER_OPCODE_LOAD_PAYLOAD)
      return false;

   fs_reg reg = this->src[0];
357
   if (reg.file != VGRF || reg.offset != 0 || reg.stride != 1)
358 359
      return false;

360
   if (grf_alloc.sizes[reg.nr] * REG_SIZE != this->size_written)
361 362
      return false;

363 364 365
   for (int i = 0; i < this->sources; i++) {
      reg.type = this->src[i].type;
      if (!this->src[i].equals(reg))
366
         return false;
367 368

      if (i < this->header_size) {
369
         reg.offset += REG_SIZE;
370
      } else {
371
         reg = horiz_offset(reg, this->exec_size);
372
      }
373
   }
374 375 376 377

   return true;
}

378
bool
379
fs_inst::can_do_source_mods(const struct gen_device_info *devinfo)
380
{
381
   if (devinfo->gen == 6 && is_math())
382 383
      return false;

384
   if (is_send_from_grf())
385 386
      return false;

387
   if (!backend_instruction::can_do_source_mods())
388 389
      return false;

390 391 392
   return true;
}

393 394 395 396 397 398 399 400 401 402 403 404
bool
fs_inst::can_change_types() const
{
   return dst.type == src[0].type &&
          !src[0].abs && !src[0].negate && !saturate &&
          (opcode == BRW_OPCODE_MOV ||
           (opcode == BRW_OPCODE_SEL &&
            dst.type == src[1].type &&
            predicate != BRW_PREDICATE_NONE &&
            !src[1].abs && !src[1].negate));
}

405 406 407 408
void
fs_reg::init()
{
   memset(this, 0, sizeof(*this));
409
   type = BRW_REGISTER_TYPE_UD;
410
   stride = 1;
411 412 413 414 415 416 417 418 419
}

/** Generic unset register constructor. */
fs_reg::fs_reg()
{
   init();
   this->file = BAD_FILE;
}

420
fs_reg::fs_reg(struct ::brw_reg reg) :
421
   backend_reg(reg)
422
{
423
   this->offset = 0;
424
   this->stride = 1;
425 426 427 428 429 430
   if (this->file == IMM &&
       (this->type != BRW_REGISTER_TYPE_V &&
        this->type != BRW_REGISTER_TYPE_UV &&
        this->type != BRW_REGISTER_TYPE_VF)) {
      this->stride = 0;
   }
431 432 433 434 435
}

bool
fs_reg::equals(const fs_reg &r) const
{
436
   return (this->backend_reg::equals(r) &&
437
           stride == r.stride);
438 439
}

440 441 442 443 444 445
bool
fs_reg::is_contiguous() const
{
   return stride == 1;
}

446 447 448
unsigned
fs_reg::component_size(unsigned width) const
{
449
   const unsigned stride = ((file != ARF && file != FIXED_GRF) ? this->stride :
450 451
                            hstride == 0 ? 0 :
                            1 << (hstride - 1));
452 453 454
   return MAX2(width * stride, 1) * type_sz(type);
}

455 456
extern "C" int
type_size_scalar(const struct glsl_type *type)
457 458 459 460 461 462 463 464
{
   unsigned int size, i;

   switch (type->base_type) {
   case GLSL_TYPE_UINT:
   case GLSL_TYPE_INT:
   case GLSL_TYPE_FLOAT:
   case GLSL_TYPE_BOOL:
465
      return type->components();
466 467 468 469
   case GLSL_TYPE_UINT16:
   case GLSL_TYPE_INT16:
   case GLSL_TYPE_FLOAT16:
      return DIV_ROUND_UP(type->components(), 2);
470
   case GLSL_TYPE_DOUBLE:
471 472
   case GLSL_TYPE_UINT64:
   case GLSL_TYPE_INT64:
473
      return type->components() * 2;
474
   case GLSL_TYPE_ARRAY:
475
      return type_size_scalar(type->fields.array) * type->length;
476 477 478
   case GLSL_TYPE_STRUCT:
      size = 0;
      for (i = 0; i < type->length; i++) {
479
	 size += type_size_scalar(type->fields.structure[i].type);
480 481 482 483 484 485 486
      }
      return size;
   case GLSL_TYPE_SAMPLER:
      /* Samplers take up no register space, since they're baked in at
       * link time.
       */
      return 0;
487 488
   case GLSL_TYPE_ATOMIC_UINT:
      return 0;
489 490
   case GLSL_TYPE_SUBROUTINE:
      return 1;
491
   case GLSL_TYPE_IMAGE:
492
      return BRW_IMAGE_PARAM_SIZE;
493 494
   case GLSL_TYPE_VOID:
   case GLSL_TYPE_ERROR:
Ian Romanick's avatar
Ian Romanick committed
495
   case GLSL_TYPE_INTERFACE:
496
   case GLSL_TYPE_FUNCTION:
497
      unreachable("not reached");
498
   }
499 500

   return 0;
501 502
}

503 504 505 506 507 508
/**
 * Create a MOV to read the timestamp register.
 *
 * The caller is responsible for emitting the MOV.  The return value is
 * the destination of the MOV, with extra parameters set.
 */
509
fs_reg
510
fs_visitor::get_timestamp(const fs_builder &bld)
511
{
512
   assert(devinfo->gen >= 7);
513

514
   fs_reg ts = fs_reg(retype(brw_vec4_reg(BRW_ARCHITECTURE_REGISTER_FILE,
515 516 517 518
                                          BRW_ARF_TIMESTAMP,
                                          0),
                             BRW_REGISTER_TYPE_UD));

Matt Turner's avatar
Matt Turner committed
519
   fs_reg dst = fs_reg(VGRF, alloc.allocate(1), BRW_REGISTER_TYPE_UD);
520

521 522
   /* We want to read the 3 fields we care about even if it's not enabled in
    * the dispatch.
523
    */
524
   bld.group(4, 0).exec_all().MOV(dst, ts);
525 526 527 528 529 530 531

   return dst;
}

void
fs_visitor::emit_shader_time_begin()
{
532 533 534 535 536 537
   /* We want only the low 32 bits of the timestamp.  Since it's running
    * at the GPU clock rate of ~1.2ghz, it will roll over every ~3 seconds,
    * which is plenty of time for our purposes.  It is identical across the
    * EUs, but since it's tracking GPU core speed it will increment at a
    * varying rate as render P-states change.
    */
538 539
   shader_start_time = component(
      get_timestamp(bld.annotate("shader time start")), 0);
540 541 542 543 544
}

void
fs_visitor::emit_shader_time_end()
{
545 546 547
   /* Insert our code just before the final SEND with EOT. */
   exec_node *end = this->instructions.get_tail();
   assert(end && ((fs_inst *) end)->eot);
548 549
   const fs_builder ibld = bld.annotate("shader time end")
                              .exec_all().at(NULL, end);
550
   const fs_reg timestamp = get_timestamp(ibld);
551

552 553 554 555 556 557 558
   /* We only use the low 32 bits of the timestamp - see
    * emit_shader_time_begin()).
    *
    * We could also check if render P-states have changed (or anything
    * else that might disrupt timing) by setting smear to 2 and checking if
    * that field is != 0.
    */
559
   const fs_reg shader_end_time = component(timestamp, 0);
560

561 562 563
   /* Check that there weren't any timestamp reset events (assuming these
    * were the only two timestamp reads that happened).
    */
564
   const fs_reg reset = component(timestamp, 2);
565
   set_condmod(BRW_CONDITIONAL_Z,
566
               ibld.AND(ibld.null_reg_ud(), reset, brw_imm_ud(1u)));
567
   ibld.IF(BRW_PREDICATE_NORMAL);
568

569
   fs_reg start = shader_start_time;
570
   start.negate = true;
571 572 573
   const fs_reg diff = component(fs_reg(VGRF, alloc.allocate(1),
                                        BRW_REGISTER_TYPE_UD),
                                 0);
574 575
   const fs_builder cbld = ibld.group(1, 0);
   cbld.group(1, 0).ADD(diff, start, shader_end_time);
576 577 578 579 580

   /* If there were no instructions between the two timestamp gets, the diff
    * is 2 cycles.  Remove that overhead, so I can forget about that when
    * trying to determine the time taken for single instructions.
    */
581
   cbld.ADD(diff, diff, brw_imm_ud(-2u));
582
   SHADER_TIME_ADD(cbld, 0, diff);
583
   SHADER_TIME_ADD(cbld, 1, brw_imm_ud(1u));
584
   ibld.emit(BRW_OPCODE_ELSE);
585
   SHADER_TIME_ADD(cbld, 2, brw_imm_ud(1u));
586
   ibld.emit(BRW_OPCODE_ENDIF);
587 588
}

589 590
void
fs_visitor::SHADER_TIME_ADD(const fs_builder &bld,
591
                            int shader_time_subindex,
592
                            fs_reg value)
593
{
594
   int index = shader_time_index * 3 + shader_time_subindex;
595
   struct brw_reg offset = brw_imm_d(index * BRW_SHADER_TIME_STRIDE);
596

597 598
   fs_reg payload;
   if (dispatch_width == 8)
599
      payload = vgrf(glsl_type::uvec2_type);
600
   else
601
      payload = vgrf(glsl_type::uint_type);
602

603
   bld.emit(SHADER_OPCODE_SHADER_TIME_ADD, fs_reg(), payload, offset, value);
604 605
}

606
void
607
fs_visitor::vfail(const char *format, va_list va)
608
{
609
   char *msg;
610

611 612
   if (failed)
      return;
613

614 615 616
   failed = true;

   msg = ralloc_vasprintf(mem_ctx, format, va);
617
   msg = ralloc_asprintf(mem_ctx, "%s compile failed: %s\n", stage_abbrev, msg);
618 619 620

   this->fail_msg = msg;

621
   if (debug_enabled) {
622
      fprintf(stderr, "%s",  msg);
623 624 625
   }
}

626 627 628 629 630 631 632 633 634 635 636
void
fs_visitor::fail(const char *format, ...)
{
   va_list va;

   va_start(va, format);
   vfail(format, va);
   va_end(va);
}

/**
637 638
 * Mark this program as impossible to compile with dispatch width greater
 * than n.
639 640
 *
 * During the SIMD8 compile (which happens first), we can detect and flag
641 642
 * things that are unsupported in SIMD16+ mode, so the compiler can skip the
 * SIMD16+ compile altogether.
643
 *
644 645
 * During a compile of dispatch width greater than n (if one happens anyway),
 * this just calls fail().
646 647
 */
void
648
fs_visitor::limit_dispatch_width(unsigned n, const char *msg)
649
{
650
   if (dispatch_width > n) {
651
      fail("%s", msg);
652
   } else {
653
      max_dispatch_width = n;
654
      compiler->shader_perf_log(log_data,
655 656
                                "Shader dispatch width limited to SIMD%d: %s",
                                n, msg);
657 658 659
   }
}

660 661 662 663 664 665 666 667 668
/**
 * Returns true if the instruction has a flag that means it won't
 * update an entire destination register.
 *
 * For example, dead code elimination and live variable analysis want to know
 * when a write to a variable screens off any preceding values that were in
 * it.
 */
bool
669
fs_inst::is_partial_write() const
670
{
671
   return ((this->predicate && this->opcode != BRW_OPCODE_SEL) ||
672
           (this->exec_size * type_sz(this->dst.type)) < 32 ||
673
           !this->dst.is_contiguous() ||
674
           this->dst.offset % REG_SIZE != 0);
675 676
}

677 678 679
unsigned
fs_inst::components_read(unsigned i) const
{
680 681 682 683
   /* Return zero if the source is not present. */
   if (src[i].file == BAD_FILE)
      return 0;

684 685 686 687 688 689 690 691 692 693 694 695
   switch (opcode) {
   case FS_OPCODE_LINTERP:
      if (i == 0)
         return 2;
      else
         return 1;

   case FS_OPCODE_PIXEL_X:
   case FS_OPCODE_PIXEL_Y:
      assert(i == 0);
      return 2;

696
   case FS_OPCODE_FB_WRITE_LOGICAL:
697
      assert(src[FB_WRITE_LOGICAL_SRC_COMPONENTS].file == IMM);
698 699
      /* First/second FB write color. */
      if (i < 2)
700
         return src[FB_WRITE_LOGICAL_SRC_COMPONENTS].ud;
701 702 703
      else
         return 1;

704 705 706 707 708 709 710
   case SHADER_OPCODE_TEX_LOGICAL:
   case SHADER_OPCODE_TXD_LOGICAL:
   case SHADER_OPCODE_TXF_LOGICAL:
   case SHADER_OPCODE_TXL_LOGICAL:
   case SHADER_OPCODE_TXS_LOGICAL:
   case FS_OPCODE_TXB_LOGICAL:
   case SHADER_OPCODE_TXF_CMS_LOGICAL:
711
   case SHADER_OPCODE_TXF_CMS_W_LOGICAL:
712 713 714 715 716
   case SHADER_OPCODE_TXF_UMS_LOGICAL:
   case SHADER_OPCODE_TXF_MCS_LOGICAL:
   case SHADER_OPCODE_LOD_LOGICAL:
   case SHADER_OPCODE_TG4_LOGICAL:
   case SHADER_OPCODE_TG4_OFFSET_LOGICAL:
717
   case SHADER_OPCODE_SAMPLEINFO_LOGICAL:
718 719
      assert(src[TEX_LOGICAL_SRC_COORD_COMPONENTS].file == IMM &&
             src[TEX_LOGICAL_SRC_GRAD_COMPONENTS].file == IMM);
720
      /* Texture coordinates. */
721 722
      if (i == TEX_LOGICAL_SRC_COORDINATE)
         return src[TEX_LOGICAL_SRC_COORD_COMPONENTS].ud;
723
      /* Texture derivatives. */
724 725 726
      else if ((i == TEX_LOGICAL_SRC_LOD || i == TEX_LOGICAL_SRC_LOD2) &&
               opcode == SHADER_OPCODE_TXD_LOGICAL)
         return src[TEX_LOGICAL_SRC_GRAD_COMPONENTS].ud;
727
      /* Texture offset. */
728
      else if (i == TEX_LOGICAL_SRC_TG4_OFFSET)
729
         return 2;
730
      /* MCS */
731
      else if (i == TEX_LOGICAL_SRC_MCS && opcode == SHADER_OPCODE_TXF_CMS_W_LOGICAL)
732
         return 2;
733 734 735
      else
         return 1;

736 737 738 739 740
   case SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL:
   case SHADER_OPCODE_TYPED_SURFACE_READ_LOGICAL:
      assert(src[3].file == IMM);
      /* Surface coordinates. */
      if (i == 0)
741
         return src[3].ud;
742 743 744 745 746 747 748 749 750 751 752 753
      /* Surface operation source (ignored for reads). */
      else if (i == 1)
         return 0;
      else
         return 1;

   case SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL:
   case SHADER_OPCODE_TYPED_SURFACE_WRITE_LOGICAL:
      assert(src[3].file == IMM &&
             src[4].file == IMM);
      /* Surface coordinates. */
      if (i == 0)
754
         return src[3].ud;
755 756
      /* Surface operation source. */
      else if (i == 1)
757
         return src[4].ud;
758 759 760
      else
         return 1;

761 762 763 764 765 766 767 768 769 770 771 772
   case SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL:
      /* Scattered logical opcodes use the following params:
       * src[0] Surface coordinates
       * src[1] Surface operation source (ignored for reads)
       * src[2] Surface
       * src[3] IMM with always 1 dimension.
       * src[4] IMM with arg bitsize for scattered read/write 8, 16, 32
       */
      assert(src[3].file == IMM &&
             src[4].file == IMM);
      return i == 1 ? 0 : 1;

773 774 775 776 777
   case SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL:
      assert(src[3].file == IMM &&
             src[4].file == IMM);
      return 1;

778 779 780 781
   case SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL:
   case SHADER_OPCODE_TYPED_ATOMIC_LOGICAL: {
      assert(src[3].file == IMM &&
             src[4].file == IMM);
782
      const unsigned op = src[4].ud;
783 784
      /* Surface coordinates. */
      if (i == 0)
785
         return src[3].ud;
786 787 788 789 790 791 792 793 794 795
      /* Surface operation source. */
      else if (i == 1 && op == BRW_AOP_CMPWR)
         return 2;
      else if (i == 1 && (op == BRW_AOP_INC || op == BRW_AOP_DEC ||
                          op == BRW_AOP_PREDEC))
         return 0;
      else
         return 1;
   }

796 797 798 799 800
   default:
      return 1;
   }
}

801
unsigned
802
fs_inst::size_read(int arg) const
803
{
804 805
   switch (opcode) {
   case FS_OPCODE_FB_WRITE:
806
   case FS_OPCODE_FB_READ:
807
   case SHADER_OPCODE_URB_WRITE_SIMD8:
808 809 810
   case SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT:
   case SHADER_OPCODE_URB_WRITE_SIMD8_MASKED:
   case SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT:
811
   case SHADER_OPCODE_URB_READ_SIMD8:
812
   case SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT:
813 814 815 816 817 818 819
   case SHADER_OPCODE_UNTYPED_ATOMIC:
   case SHADER_OPCODE_UNTYPED_SURFACE_READ:
   case SHADER_OPCODE_UNTYPED_SURFACE_WRITE:
   case SHADER_OPCODE_TYPED_ATOMIC:
   case SHADER_OPCODE_TYPED_SURFACE_READ:
   case SHADER_OPCODE_TYPED_SURFACE_WRITE:
   case FS_OPCODE_INTERPOLATE_AT_PER_SLOT_OFFSET:
820
   case SHADER_OPCODE_BYTE_SCATTERED_WRITE:
821
   case SHADER_OPCODE_BYTE_SCATTERED_READ:
822
      if (arg == 0)
823
         return mlen * REG_SIZE;
824 825
      break;

826 827 828
   case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD_GEN7:
      /* The payload is actually stored in src1 */
      if (arg == 1)
829
         return mlen * REG_SIZE;
830 831
      break;

832
   case FS_OPCODE_LINTERP:
833
      if (arg == 1)
834
         return 16;
835 836
      break;

837 838
   case SHADER_OPCODE_LOAD_PAYLOAD:
      if (arg < this->header_size)
839
         return REG_SIZE;
840 841
      break;

842
   case CS_OPCODE_CS_TERMINATE:
843
   case SHADER_OPCODE_BARRIER:
844
      return REG_SIZE;
845

846 847 848
   case SHADER_OPCODE_MOV_INDIRECT:
      if (arg == 0) {
         assert(src[2].file == IMM);
849
         return src[2].ud;
850 851 852
      }
      break;

853
   default:
Matt Turner's avatar
Matt Turner committed
854
      if (is_tex() && arg == 0 && src[0].file == VGRF)
855
         return mlen * REG_SIZE;
856
      break;
857 858 859 860 861
   }

   switch (src[arg].file) {
   case UNIFORM:
   case IMM:
862
      return components_read(arg) * type_sz(src[arg].type);
863
   case BAD_FILE:
864 865
   case ARF:
   case FIXED_GRF:
Matt Turner's avatar
Matt Turner committed
866
   case VGRF:
867
   case ATTR:
868
      return components_read(arg) * src[arg].component_size(exec_size);
869 870
   case MRF:
      unreachable("MRF registers are not allowed as sources");
871
   }
872
   return 0;
873 874
}

875 876 877 878 879 880 881 882 883 884 885 886
namespace {
   /* Return the subset of flag registers that an instruction could
    * potentially read or write based on the execution controls and flag
    * subregister number of the instruction.
    */
   unsigned
   flag_mask(const fs_inst *inst)
   {
      const unsigned start = inst->flag_subreg * 16 + inst->group;
      const unsigned end = start + inst->exec_size;
      return ((1 << DIV_ROUND_UP(end, 8)) - 1) & ~((1 << (start / 8)) - 1);
   }
887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904

   unsigned
   bit_mask(unsigned n)
   {
      return (n >= CHAR_BIT * sizeof(bit_mask(n)) ? ~0u : (1u << n) - 1);
   }

   unsigned
   flag_mask(const fs_reg &r, unsigned sz)
   {
      if (r.file == ARF) {
         const unsigned start = (r.nr - BRW_ARF_FLAG) * 4 + r.subnr;
         const unsigned end = start + sz;
         return bit_mask(end) & ~bit_mask(start);
      } else {
         return 0;
      }
   }
905 906 907
}

unsigned
908
fs_inst::flags_read(const gen_device_info *devinfo) const
909
{
910 911 912 913 914 915 916 917 918 919
   if (predicate == BRW_PREDICATE_ALIGN1_ANYV ||
       predicate == BRW_PREDICATE_ALIGN1_ALLV) {
      /* The vertical predication modes combine corresponding bits from
       * f0.0 and f1.0 on Gen7+, and f0.0 and f0.1 on older hardware.
       */
      const unsigned shift = devinfo->gen >= 7 ? 4 : 2;
      return flag_mask(this) << shift | flag_mask(this);
   } else if (predicate) {
      return flag_mask(this);
   } else {
920 921 922 923 924
      unsigned mask = 0;
      for (int i = 0; i < sources; i++) {
         mask |= flag_mask(src[i], size_read(i));
      }
      return mask;
925
   }
926 927
}

928 929
unsigned
fs_inst::flags_written() const
930
{
931 932 933 934 935 936
   if ((conditional_mod && (opcode != BRW_OPCODE_SEL &&
                            opcode != BRW_OPCODE_IF &&
                            opcode != BRW_OPCODE_WHILE)) ||
       opcode == FS_OPCODE_MOV_DISPATCH_TO_FLAGS) {
      return flag_mask(this);
   } else {
937
      return flag_mask(dst, size_written);
938
   }
939 940
}

941 942 943 944 945 946 947
/**
 * Returns how many MRFs an FS opcode will write over.
 *
 * Note that this is not the 0 or 1 implied writes in an actual gen
 * instruction -- the FS opcodes often generate MOVs in addition.
 */
int
948
fs_visitor::implied_mrf_writes(fs_inst *inst) const
949 950 951 952
{
   if (inst->mlen == 0)
      return 0;

953 954 955
   if (inst->base_mrf == -1)
      return 0;

956
   switch (inst->opcode) {
957 958 959 960 961 962 963
   case SHADER_OPCODE_RCP:
   case SHADER_OPCODE_RSQ:
   case SHADER_OPCODE_SQRT:
   case SHADER_OPCODE_EXP2:
   case SHADER_OPCODE_LOG2:
   case SHADER_OPCODE_SIN:
   case SHADER_OPCODE_COS:
964
      return 1 * dispatch_width / 8;
965
   case SHADER_OPCODE_POW:
966 967
   case SHADER_OPCODE_INT_QUOTIENT:
   case SHADER_OPCODE_INT_REMAINDER:
968
      return 2 * dispatch_width / 8;
969
   case SHADER_OPCODE_TEX:
970
   case FS_OPCODE_TXB:
971 972
   case SHADER_OPCODE_TXD:
   case SHADER_OPCODE_TXF:
973
   case SHADER_OPCODE_TXF_CMS:
974
   case SHADER_OPCODE_TXF_MCS:
Chris Forbes's avatar
Chris Forbes committed
975
   case SHADER_OPCODE_TG4:
976
   case SHADER_OPCODE_TG4_OFFSET:
977 978
   case SHADER_OPCODE_TXL:
   case SHADER_OPCODE_TXS:
979
   case SHADER_OPCODE_LOD:
980
   case SHADER_OPCODE_SAMPLEINFO:
981 982 983
      return 1;
   case FS_OPCODE_FB_WRITE:
      return 2;
984
   case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD:
985
   case SHADER_OPCODE_GEN4_SCRATCH_READ:
986
      return 1;
987
   case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_GEN4:
988
      return inst->mlen;
989
   case SHADER_OPCODE_GEN4_SCRATCH_WRITE:
990
      return inst->mlen;
991
   default:
992
      unreachable("not reached");
993 994 995
   }
}

996 997 998 999
fs_reg
fs_visitor::vgrf(const glsl_type *const type)
{
   int reg_width = dispatch_width / 8;
Matt Turner's avatar
Matt Turner committed
1000
   return fs_reg(VGRF, alloc.allocate(type_size_scalar(type) * reg_width),
1001
                 brw_type_for_base_type(type));
1002 1003
}

1004
fs_reg::fs_reg(enum brw_reg_file file, int nr)
1005
{
1006
   init();
1007
   this->file = file;
1008
   this->nr = nr;
1009
   this->type = BRW_REGISTER_TYPE_F;
1010
   this->stride = (file == UNIFORM ? 0 : 1);
1011 1012
}

1013
fs_reg::fs_reg(enum brw_reg_file file, int nr, enum brw_reg_type type)
1014 1015 1016
{
   init();
   this->file = file;
1017
   this->nr = nr;
1018
   this->type = type;
1019
   this->stride = (file == UNIFORM ? 0 : 1);
1020 1021
}

1022
/* For SIMD16, we need to follow from the uniform setup of SIMD8 dispatch.
1023 1024 1025
 * This brings in those uniform definitions
 */
void
1026
fs_visitor::import_uniforms(fs_visitor *v)
1027
{
1028
   this->push_constant_loc = v->push_constant_loc;
1029
   this->pull_constant_loc = v->pull_constant_loc;
1030
   this->uniforms = v->uniforms;
1031
   this->subgroup_id = v->subgroup_id;
1032 1033
}

1034 1035
void
fs_visitor::emit_fragcoord_interpolation(fs_reg wpos)
1036
{
1037
   assert(stage == MESA_SHADER_FRAGMENT);
1038 1039

   /* gl_FragCoord.x */
1040
   bld.MOV(wpos, this->pixel_x);
1041
   wpos = offset(wpos, bld, 1);
1042 1043

   /* gl_FragCoord.y */
1044
   bld.MOV(wpos, this->pixel_y);
1045
   wpos = offset(wpos, bld, 1);
1046 1047

   /* gl_FragCoord.z */
1048
   if (devinfo->gen >= 6) {