nir.h 129 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
/*
 * Copyright © 2014 Connor Abbott
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Connor Abbott (cwabbott0@gmail.com)
 *
 */

28 29
#ifndef NIR_H
#define NIR_H
30 31

#include "util/hash_table.h"
Emil Velikov's avatar
Emil Velikov committed
32
#include "compiler/glsl/list.h"
33
#include "GL/gl.h" /* GLenum */
34
#include "util/list.h"
35
#include "util/ralloc.h"
36
#include "util/set.h"
37
#include "util/bitscan.h"
38
#include "util/bitset.h"
39
#include "util/macros.h"
40
#include "compiler/nir_types.h"
41
#include "compiler/shader_enums.h"
42
#include "compiler/shader_info.h"
43 44
#include <stdio.h>

45
#ifndef NDEBUG
46
#include "util/debug.h"
47
#endif /* NDEBUG */
48

49 50
#include "nir_opcodes.h"

51 52 53 54
#if defined(_WIN32) && !defined(snprintf)
#define snprintf _snprintf
#endif

55 56 57 58
#ifdef __cplusplus
extern "C" {
#endif

59 60
#define NIR_FALSE 0u
#define NIR_TRUE (~0u)
61
#define NIR_MAX_VEC_COMPONENTS 4
62
#define NIR_MAX_MATRIX_COLUMNS 4
63
#define NIR_STREAM_PACKED (1 << 8)
64
typedef uint8_t nir_component_mask_t;
65
typedef unsigned short GLenum16;
66

67 68 69 70 71 72 73 74
/** Defines a cast function
 *
 * This macro defines a cast function from in_type to out_type where
 * out_type is some structure type that contains a field of type out_type.
 *
 * Note that you have to be a bit careful as the generated cast function
 * destroys constness.
 */
75 76 77 78 79 80 81
#define NIR_DEFINE_CAST(name, in_type, out_type, field, \
                        type_field, type_value)         \
static inline out_type *                                \
name(const in_type *parent)                             \
{                                                       \
   assert(parent && parent->type_field == type_value);  \
   return exec_node_data(out_type, parent, field);      \
82 83
}

84
struct nir_function;
85
struct nir_shader;
86
struct nir_instr;
87
struct nir_builder;
88 89 90 91 92 93 94 95


/**
 * Description of built-in state associated with a uniform
 *
 * \sa nir_variable::state_slots
 */
typedef struct {
96
   gl_state_index16 tokens[STATE_LENGTH];
97 98 99 100
   int swizzle;
} nir_state_slot;

typedef enum {
101 102
   nir_var_shader_in       = (1 << 0),
   nir_var_shader_out      = (1 << 1),
103
   nir_var_shader_temp     = (1 << 2),
104
   nir_var_function_temp   = (1 << 3),
105
   nir_var_uniform         = (1 << 4),
106
   nir_var_mem_ubo         = (1 << 5),
107
   nir_var_system_value    = (1 << 6),
108
   nir_var_mem_ssbo        = (1 << 7),
109
   nir_var_mem_shared      = (1 << 8),
110
   nir_var_mem_global      = (1 << 9),
111
   nir_var_all             = (1 << 10) - 1,
112 113
} nir_variable_mode;

114 115 116 117 118 119 120 121 122 123
/**
 * Rounding modes.
 */
typedef enum {
   nir_rounding_mode_undef = 0,
   nir_rounding_mode_rtne  = 1, /* round to nearest even */
   nir_rounding_mode_ru    = 2, /* round up */
   nir_rounding_mode_rd    = 3, /* round down */
   nir_rounding_mode_rtz   = 4, /* round towards zero */
} nir_rounding_mode;
124 125

typedef union {
126 127 128 129 130 131 132 133 134 135 136
   bool b;
   float f32;
   double f64;
   int8_t i8;
   uint8_t u8;
   int16_t i16;
   uint16_t u16;
   int32_t i32;
   uint32_t u32;
   int64_t i64;
   uint64_t u64;
137
} nir_const_value;
138

139 140 141 142 143 144
#define nir_const_value_to_array(arr, c, components, m) \
{ \
   for (unsigned i = 0; i < components; ++i) \
      arr[i] = c[i].m; \
} while (false)

145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216
static inline nir_const_value
nir_const_value_for_raw_uint(uint64_t x, unsigned bit_size)
{
   nir_const_value v;
   memset(&v, 0, sizeof(v));

   switch (bit_size) {
   case 1:  v.b   = x;  break;
   case 8:  v.u8  = x;  break;
   case 16: v.u16 = x;  break;
   case 32: v.u32 = x;  break;
   case 64: v.u64 = x;  break;
   default:
      unreachable("Invalid bit size");
   }

   return v;
}

static inline nir_const_value
nir_const_value_for_int(int64_t i, unsigned bit_size)
{
   nir_const_value v;
   memset(&v, 0, sizeof(v));

   assert(bit_size <= 64);
   if (bit_size < 64) {
      assert(i >= (-(1ll << (bit_size - 1))));
      assert(i < (1ll << (bit_size - 1)));
   }

   return nir_const_value_for_raw_uint(i, bit_size);
}

static inline nir_const_value
nir_const_value_for_uint(uint64_t u, unsigned bit_size)
{
   nir_const_value v;
   memset(&v, 0, sizeof(v));

   assert(bit_size <= 64);
   if (bit_size < 64)
      assert(u < (1ull << bit_size));

   return nir_const_value_for_raw_uint(u, bit_size);
}

static inline nir_const_value
nir_const_value_for_bool(bool b, unsigned bit_size)
{
   /* Booleans use a 0/-1 convention */
   return nir_const_value_for_int(-(int)b, bit_size);
}

/* This one isn't inline because it requires half-float conversion */
nir_const_value nir_const_value_for_float(double b, unsigned bit_size);

static inline int64_t
nir_const_value_as_int(nir_const_value value, unsigned bit_size)
{
   switch (bit_size) {
   /* int1_t uses 0/-1 convention */
   case 1:  return -(int)value.b;
   case 8:  return value.i8;
   case 16: return value.i16;
   case 32: return value.i32;
   case 64: return value.i64;
   default:
      unreachable("Invalid bit size");
   }
}

Rob Clark's avatar
Rob Clark committed
217
static inline uint64_t
218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244
nir_const_value_as_uint(nir_const_value value, unsigned bit_size)
{
   switch (bit_size) {
   case 1:  return value.b;
   case 8:  return value.u8;
   case 16: return value.u16;
   case 32: return value.u32;
   case 64: return value.u64;
   default:
      unreachable("Invalid bit size");
   }
}

static inline bool
nir_const_value_as_bool(nir_const_value value, unsigned bit_size)
{
   int64_t i = nir_const_value_as_int(value, bit_size);

   /* Booleans of any size use 0/-1 convention */
   assert(i == 0 || i == -1);

   return i;
}

/* This one isn't inline because it requires half-float conversion */
double nir_const_value_as_float(nir_const_value value, unsigned bit_size);

245 246 247 248 249
typedef struct nir_constant {
   /**
    * Value of the constant.
    *
    * The field used to back the values supplied by the constant is determined
Jason Ekstrand's avatar
Jason Ekstrand committed
250
    * by the type associated with the \c nir_variable.  Constants may be
251 252
    * scalars, vectors, or matrices.
    */
253
   nir_const_value values[NIR_MAX_VEC_COMPONENTS];
254

Rob Clark's avatar
Rob Clark committed
255 256 257 258 259
   /* we could get this from the var->type but makes clone *much* easier to
    * not have to care about the type.
    */
   unsigned num_elements;

260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277
   /* Array elements / Structure Fields */
   struct nir_constant **elements;
} nir_constant;

/**
 * \brief Layout qualifiers for gl_FragDepth.
 *
 * The AMD/ARB_conservative_depth extensions allow gl_FragDepth to be redeclared
 * with a layout qualifier.
 */
typedef enum {
    nir_depth_layout_none, /**< No depth layout is specified. */
    nir_depth_layout_any,
    nir_depth_layout_greater,
    nir_depth_layout_less,
    nir_depth_layout_unchanged
} nir_depth_layout;

278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293
/**
 * Enum keeping track of how a variable was declared.
 */
typedef enum {
   /**
    * Normal declaration.
    */
   nir_var_declared_normally = 0,

   /**
    * Variable is implicitly generated by the compiler and should not be
    * visible via the API.
    */
   nir_var_hidden,
} nir_var_declaration_type;

294 295 296 297 298
/**
 * Either a uniform, global variable, shader input, or shader output. Based on
 * ir_variable - it should be easy to translate between the two.
 */

Rob Clark's avatar
Rob Clark committed
299
typedef struct nir_variable {
300 301 302 303 304 305 306 307 308 309 310 311 312
   struct exec_node node;

   /**
    * Declared type of the variable
    */
   const struct glsl_type *type;

   /**
    * Declared name of the variable
    */
   char *name;

   struct nir_variable_data {
313 314 315 316 317
      /**
       * Storage class of the variable.
       *
       * \sa nir_variable_mode
       */
318
      nir_variable_mode mode:10;
319 320 321 322 323 324 325 326 327 328

      /**
       * Is the variable read-only?
       *
       * This is set for variables declared as \c const, shader inputs,
       * and uniforms.
       */
      unsigned read_only:1;
      unsigned centroid:1;
      unsigned sample:1;
329
      unsigned patch:1;
330 331
      unsigned invariant:1;

332 333 334 335 336 337 338 339 340 341 342
      /**
       * Can this variable be coalesced with another?
       *
       * This is set by nir_lower_io_to_temporaries to say that any
       * copies involving this variable should stay put. Propagating it can
       * duplicate the resulting load/store, which is not wanted, and may
       * result in a load/store of the variable with an indirect offset which
       * the backend may not be able to handle.
       */
      unsigned cannot_coalesce:1;

343 344 345 346 347 348 349 350 351 352
      /**
       * When separate shader programs are enabled, only input/outputs between
       * the stages of a multi-stage separate program can be safely removed
       * from the shader interface. Other input/outputs must remains active.
       *
       * This is also used to make sure xfb varyings that are unused by the
       * fragment shader are not removed.
       */
      unsigned always_active_io:1;

353 354 355
      /**
       * Interpolation mode for shader inputs / outputs
       *
356
       * \sa glsl_interp_mode
357 358 359 360 361 362 363 364 365 366 367 368
       */
      unsigned interpolation:2;

      /**
       * If non-zero, then this variable may be packed along with other variables
       * into a single varying slot, so this offset should be applied when
       * accessing components.  For example, an offset of 1 means that the x
       * component of this variable is actually stored in component y of the
       * location specified by \c location.
       */
      unsigned location_frac:2;

369 370 371 372 373
      /**
       * If true, this variable represents an array of scalars that should
       * be tightly packed.  In other words, consecutive array elements
       * should be stored one component apart, rather than one slot apart.
       */
374
      unsigned compact:1;
375

376 377 378 379 380 381 382
      /**
       * Whether this is a fragment shader output implicitly initialized with
       * the previous contents of the specified render target at the
       * framebuffer location corresponding to this shader invocation.
       */
      unsigned fb_fetch_output:1;

383 384 385 386 387 388
      /**
       * Non-zero if this variable is considered bindless as defined by
       * ARB_bindless_texture.
       */
      unsigned bindless:1;

389 390 391 392 393
      /**
       * Was an explicit binding set in the shader?
       */
      unsigned explicit_binding:1;

394 395 396 397 398 399 400 401 402 403 404 405 406 407 408
      /**
       * Was a transfer feedback buffer set in the shader?
       */
      unsigned explicit_xfb_buffer:1;

      /**
       * Was a transfer feedback stride set in the shader?
       */
      unsigned explicit_xfb_stride:1;

      /**
       * Was an explicit offset set in the shader?
       */
      unsigned explicit_offset:1;

409 410 411 412 413 414 415 416
      /**
       * How the variable was declared.  See nir_var_declaration_type.
       *
       * This is used to detect variables generated by the compiler, so should
       * not be visible via the API.
       */
      unsigned how_declared:2;

417 418 419 420 421 422
      /**
       * \brief Layout qualifier for gl_FragDepth.
       *
       * This is not equal to \c ir_depth_layout_none if and only if this
       * variable is \c gl_FragDepth and a layout qualifier is specified.
       */
423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448
      nir_depth_layout depth_layout:3;

      /**
       * Vertex stream output identifier.
       *
       * For packed outputs, NIR_STREAM_PACKED is set and bits [2*i+1,2*i]
       * indicate the stream of the i-th component.
       */
      unsigned stream:9;

      /**
       * output index for dual source blending.
       */
      unsigned index;

      /**
       * Descriptor set binding for sampler or UBO.
       */
      int descriptor_set:5;

      /**
       * Initial binding point for a sampler or UBO.
       *
       * For array types, this represents the binding point for the first element.
       */
      unsigned binding:5;
449 450 451 452 453 454 455 456 457 458 459 460 461 462

      /**
       * Storage location of the base of this variable
       *
       * The precise meaning of this field depends on the nature of the variable.
       *
       *   - Vertex shader input: one of the values from \c gl_vert_attrib.
       *   - Vertex shader output: one of the values from \c gl_varying_slot.
       *   - Geometry shader input: one of the values from \c gl_varying_slot.
       *   - Geometry shader output: one of the values from \c gl_varying_slot.
       *   - Fragment shader input: one of the values from \c gl_varying_slot.
       *   - Fragment shader output: one of the values from \c gl_frag_result.
       *   - Uniforms: Per-stage uniform slot number for default uniform block.
       *   - Uniforms: Index within the uniform block definition for UBO members.
463
       *   - Non-UBO Uniforms: uniform slot number.
464 465 466 467 468 469 470 471
       *   - Other: This field is not currently used.
       *
       * If the variable is a uniform, shader input, or shader output, and the
       * slot has not been assigned, the value will be -1.
       */
      int location;

      /**
472 473
       * The actual location of the variable in the IR. Only valid for inputs,
       * outputs, and uniforms (including samplers and images).
474
       */
475
      unsigned driver_location;
476 477

      /**
478
       * Location an atomic counter or transform feedback is stored at.
479
       */
480
      unsigned offset;
481

482 483 484 485 486
      union {
         /**
          * ARB_shader_image_load_store qualifiers.
          */
         struct {
487
            enum gl_access_qualifier access:8;
488 489

            /** Image internal format if specified explicitly, otherwise GL_NONE. */
490
            GLenum16 format;
491 492 493 494 495 496 497 498 499 500 501 502 503 504
         } image;

         struct {
            /**
             * Transform feedback buffer.
             */
            uint16_t buffer:2;

            /**
             * Transform feedback stride.
             */
            uint16_t stride;
         } xfb;
      };
505 506
   } data;

507 508 509
   /* Number of nir_variable_data members */
   unsigned num_members;

510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527
   /**
    * Built-in state that backs this uniform
    *
    * Once set at variable creation, \c state_slots must remain invariant.
    * This is because, ideally, this array would be shared by all clones of
    * this variable in the IR tree.  In other words, we'd really like for it
    * to be a fly-weight.
    *
    * If the variable is not a uniform, \c num_state_slots will be zero and
    * \c state_slots will be \c NULL.
    */
   /*@{*/
   unsigned num_state_slots;    /**< Number of state slots used */
   nir_state_slot *state_slots;  /**< State descriptors. */
   /*@}*/

   /**
    * Constant expression assigned in the initializer of the variable
528 529 530 531
    *
    * This field should only be used temporarily by creators of NIR shaders
    * and then lower_constant_initializers can be used to get rid of them.
    * Most of the rest of NIR ignores this field or asserts that it's NULL.
532 533 534 535 536 537 538 539 540 541
    */
   nir_constant *constant_initializer;

   /**
    * For variables that are in an interface block or are an instance of an
    * interface block, this is the \c GLSL_TYPE_INTERFACE type for that block.
    *
    * \sa ir_variable::location
    */
   const struct glsl_type *interface_type;
542 543 544 545 546 547 548 549 550 551

   /**
    * Description of per-member data for per-member struct variables
    *
    * This is used for variables which are actually an amalgamation of
    * multiple entities such as a struct of built-in values or a struct of
    * inputs each with their own layout specifier.  This is only allowed on
    * variables with a struct or array of array of struct type.
    */
   struct nir_variable_data *members;
552 553
} nir_variable;

554 555
#define nir_foreach_variable(var, var_list) \
   foreach_list_typed(nir_variable, var, node, var_list)
556 557 558

#define nir_foreach_variable_safe(var, var_list) \
   foreach_list_typed_safe(nir_variable, var, node, var_list)
559

560 561 562
static inline bool
nir_variable_is_global(const nir_variable *var)
{
563
   return var->data.mode != nir_var_function_temp;
564 565
}

Rob Clark's avatar
Rob Clark committed
566
typedef struct nir_register {
567 568 569 570 571
   struct exec_node node;

   unsigned num_components; /** < number of vector components */
   unsigned num_array_elems; /** < size of array (0 for no array) */

572 573 574
   /* The bit-size of each channel; must be one of 8, 16, 32, or 64 */
   uint8_t bit_size;

575
   /** generic register index. */
576 577 578 579 580
   unsigned index;

   /** only for debug purposes, can be NULL */
   const char *name;

581
   /** set of nir_srcs where this register is used (read from) */
582
   struct list_head uses;
583

584
   /** set of nir_dests where this register is defined (written to) */
585
   struct list_head defs;
586

587
   /** set of nir_ifs where this register is used as a condition */
588
   struct list_head if_uses;
589 590
} nir_register;

591 592 593 594 595
#define nir_foreach_register(reg, reg_list) \
   foreach_list_typed(nir_register, reg, node, reg_list)
#define nir_foreach_register_safe(reg, reg_list) \
   foreach_list_typed_safe(nir_register, reg, node, reg_list)

Ian Romanick's avatar
Ian Romanick committed
596
typedef enum PACKED {
597
   nir_instr_type_alu,
598
   nir_instr_type_deref,
599
   nir_instr_type_call,
600
   nir_instr_type_tex,
601 602 603 604 605
   nir_instr_type_intrinsic,
   nir_instr_type_load_const,
   nir_instr_type_jump,
   nir_instr_type_ssa_undef,
   nir_instr_type_phi,
606
   nir_instr_type_parallel_copy,
607 608
} nir_instr_type;

609
typedef struct nir_instr {
610 611
   struct exec_node node;
   struct nir_block *block;
Ian Romanick's avatar
Ian Romanick committed
612
   nir_instr_type type;
613

614 615 616 617
   /* A temporary for optimization and analysis passes to use for storing
    * flags.  For instance, DCE uses this to store the "dead/live" info.
    */
   uint8_t pass_flags;
Ian Romanick's avatar
Ian Romanick committed
618 619 620

   /** generic instruction index. */
   unsigned index;
621 622
} nir_instr;

623
static inline nir_instr *
624
nir_instr_next(nir_instr *instr)
625
{
626 627 628 629 630
   struct exec_node *next = exec_node_get_next(&instr->node);
   if (exec_node_is_tail_sentinel(next))
      return NULL;
   else
      return exec_node_data(nir_instr, next, node);
631
}
632

633
static inline nir_instr *
634
nir_instr_prev(nir_instr *instr)
635
{
636 637 638 639 640
   struct exec_node *prev = exec_node_get_prev(&instr->node);
   if (exec_node_is_head_sentinel(prev))
      return NULL;
   else
      return exec_node_data(nir_instr, prev, node);
641
}
642

643
static inline bool
644
nir_instr_is_first(const nir_instr *instr)
645
{
646
   return exec_node_is_head_sentinel(exec_node_get_prev_const(&instr->node));
647 648 649
}

static inline bool
650
nir_instr_is_last(const nir_instr *instr)
651
{
652
   return exec_node_is_tail_sentinel(exec_node_get_next_const(&instr->node));
653 654
}

Rob Clark's avatar
Rob Clark committed
655
typedef struct nir_ssa_def {
656 657 658
   /** for debugging only, can be NULL */
   const char* name;

659
   /** generic SSA definition index. */
660 661
   unsigned index;

662 663 664
   /** Index into the live_in and live_out bitfields */
   unsigned live_index;

665
   /** Instruction which produces this SSA value. */
666 667
   nir_instr *parent_instr;

668
   /** set of nir_instrs where this register is used (read from) */
669
   struct list_head uses;
Jason Ekstrand's avatar
Jason Ekstrand committed
670

671
   /** set of nir_ifs where this register is used as a condition */
672
   struct list_head if_uses;
673 674

   uint8_t num_components;
675 676 677

   /* The bit-size of each channel; must be one of 8, 16, 32, or 64 */
   uint8_t bit_size;
678 679 680 681 682 683 684 685 686 687 688 689 690
} nir_ssa_def;

struct nir_src;

typedef struct {
   nir_register *reg;
   struct nir_src *indirect; /** < NULL for no indirect offset */
   unsigned base_offset;

   /* TODO use-def chain goes here */
} nir_reg_src;

typedef struct {
691 692 693
   nir_instr *parent_instr;
   struct list_head def_link;

694 695 696 697 698 699 700
   nir_register *reg;
   struct nir_src *indirect; /** < NULL for no indirect offset */
   unsigned base_offset;

   /* TODO def-use chain goes here */
} nir_reg_dest;

701 702
struct nir_if;

703
typedef struct nir_src {
704
   union {
705
      /** Instruction that consumes this value as a source. */
706 707 708 709 710 711
      nir_instr *parent_instr;
      struct nir_if *parent_if;
   };

   struct list_head use_link;

712 713 714 715 716 717 718 719
   union {
      nir_reg_src reg;
      nir_ssa_def *ssa;
   };

   bool is_ssa;
} nir_src;

720 721 722 723 724 725 726 727
static inline nir_src
nir_src_init(void)
{
   nir_src src = { { NULL } };
   return src;
}

#define NIR_SRC_INIT nir_src_init()
728

729
#define nir_foreach_use(src, reg_or_ssa_def) \
730 731
   list_for_each_entry(nir_src, src, &(reg_or_ssa_def)->uses, use_link)

732
#define nir_foreach_use_safe(src, reg_or_ssa_def) \
733 734
   list_for_each_entry_safe(nir_src, src, &(reg_or_ssa_def)->uses, use_link)

735
#define nir_foreach_if_use(src, reg_or_ssa_def) \
736 737
   list_for_each_entry(nir_src, src, &(reg_or_ssa_def)->if_uses, use_link)

738
#define nir_foreach_if_use_safe(src, reg_or_ssa_def) \
739
   list_for_each_entry_safe(nir_src, src, &(reg_or_ssa_def)->if_uses, use_link)
740

741 742 743 744 745 746 747 748 749
typedef struct {
   union {
      nir_reg_dest reg;
      nir_ssa_def ssa;
   };

   bool is_ssa;
} nir_dest;

750 751 752 753 754 755 756 757
static inline nir_dest
nir_dest_init(void)
{
   nir_dest dest = { { { NULL } } };
   return dest;
}

#define NIR_DEST_INIT nir_dest_init()
758

759
#define nir_foreach_def(dest, reg) \
760 761
   list_for_each_entry(nir_dest, dest, &(reg)->defs, reg.def_link)

762
#define nir_foreach_def_safe(dest, reg) \
763 764
   list_for_each_entry_safe(nir_dest, dest, &(reg)->defs, reg.def_link)

765 766 767
static inline nir_src
nir_src_for_ssa(nir_ssa_def *def)
{
768
   nir_src src = NIR_SRC_INIT;
769 770 771 772 773 774 775 776 777 778

   src.is_ssa = true;
   src.ssa = def;

   return src;
}

static inline nir_src
nir_src_for_reg(nir_register *reg)
{
779
   nir_src src = NIR_SRC_INIT;
780 781 782 783 784 785 786 787 788 789 790 791

   src.is_ssa = false;
   src.reg.reg = reg;
   src.reg.indirect = NULL;
   src.reg.base_offset = 0;

   return src;
}

static inline nir_dest
nir_dest_for_reg(nir_register *reg)
{
792
   nir_dest dest = NIR_DEST_INIT;
793 794 795 796 797 798

   dest.reg.reg = reg;

   return dest;
}

799 800 801 802 803 804
static inline unsigned
nir_src_bit_size(nir_src src)
{
   return src.is_ssa ? src.ssa->bit_size : src.reg.reg->bit_size;
}

805 806 807 808 809 810
static inline unsigned
nir_src_num_components(nir_src src)
{
   return src.is_ssa ? src.ssa->num_components : src.reg.reg->num_components;
}

811 812 813 814 815 816 817
static inline bool
nir_src_is_const(nir_src src)
{
   return src.is_ssa &&
          src.ssa->parent_instr->type == nir_instr_type_load_const;
}

818 819 820 821 822 823
static inline unsigned
nir_dest_bit_size(nir_dest dest)
{
   return dest.is_ssa ? dest.ssa.bit_size : dest.reg.reg->bit_size;
}

824 825 826 827 828 829
static inline unsigned
nir_dest_num_components(nir_dest dest)
{
   return dest.is_ssa ? dest.ssa.num_components : dest.reg.reg->num_components;
}

830 831
void nir_src_copy(nir_src *dest, const nir_src *src, void *instr_or_if);
void nir_dest_copy(nir_dest *dest, const nir_dest *src, nir_instr *instr);
832 833 834 835 836 837 838 839 840

typedef struct {
   nir_src src;

   /**
    * \name input modifiers
    */
   /*@{*/
   /**
Jason Ekstrand's avatar
Jason Ekstrand committed
841 842
    * For inputs interpreted as floating point, flips the sign bit. For
    * inputs interpreted as integers, performs the two's complement negation.
843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861
    */
   bool negate;

   /**
    * Clears the sign bit for floating point values, and computes the integer
    * absolute value for integers. Note that the negate modifier acts after
    * the absolute value modifier, therefore if both are set then all inputs
    * will become negative.
    */
   bool abs;
   /*@}*/

   /**
    * For each input component, says which component of the register it is
    * chosen from. Note that which elements of the swizzle are used and which
    * are ignored are based on the write mask for most opcodes - for example,
    * a statement like "foo.xzw = bar.zyx" would have a writemask of 1101b and
    * a swizzle of {2, x, 1, 0} where x means "don't care."
    */
862
   uint8_t swizzle[NIR_MAX_VEC_COMPONENTS];
863 864 865 866 867 868 869 870 871 872 873 874 875 876
} nir_alu_src;

typedef struct {
   nir_dest dest;

   /**
    * \name saturate output modifier
    *
    * Only valid for opcodes that output floating-point numbers. Clamps the
    * output to between 0.0 and 1.0 inclusive.
    */

   bool saturate;

877
   unsigned write_mask : NIR_MAX_VEC_COMPONENTS; /* ignored if dest.is_ssa is true */
878 879
} nir_alu_dest;

880 881 882 883 884
/** NIR sized and unsized types
 *
 * The values in this enum are carefully chosen so that the sized type is
 * just the unsized type OR the number of bits.
 */
885
typedef enum {
Jason Ekstrand's avatar
Jason Ekstrand committed
886
   nir_type_invalid = 0, /* Not a valid type */
887 888 889 890 891
   nir_type_int =       2,
   nir_type_uint =      4,
   nir_type_bool =      6,
   nir_type_float =     128,
   nir_type_bool1 =     1  | nir_type_bool,
892
   nir_type_bool32 =    32 | nir_type_bool,
893
   nir_type_int1 =      1  | nir_type_int,
894 895 896 897
   nir_type_int8 =      8  | nir_type_int,
   nir_type_int16 =     16 | nir_type_int,
   nir_type_int32 =     32 | nir_type_int,
   nir_type_int64 =     64 | nir_type_int,
898
   nir_type_uint1 =     1  | nir_type_uint,
899 900 901 902 903 904 905
   nir_type_uint8 =     8  | nir_type_uint,
   nir_type_uint16 =    16 | nir_type_uint,
   nir_type_uint32 =    32 | nir_type_uint,
   nir_type_uint64 =    64 | nir_type_uint,
   nir_type_float16 =   16 | nir_type_float,
   nir_type_float32 =   32 | nir_type_float,
   nir_type_float64 =   64 | nir_type_float,
906 907
} nir_alu_type;

908 909
#define NIR_ALU_TYPE_SIZE_MASK 0x79
#define NIR_ALU_TYPE_BASE_TYPE_MASK 0x86
910 911 912 913 914 915 916 917 918 919 920 921 922

static inline unsigned
nir_alu_type_get_type_size(nir_alu_type type)
{
   return type & NIR_ALU_TYPE_SIZE_MASK;
}

static inline unsigned
nir_alu_type_get_base_type(nir_alu_type type)
{
   return type & NIR_ALU_TYPE_BASE_TYPE_MASK;
}

923
static inline nir_alu_type
924
nir_get_nir_type_for_glsl_base_type(enum glsl_base_type base_type)
925
{
926
   switch (base_type) {
927
   case GLSL_TYPE_BOOL:
928
      return nir_type_bool1;
929 930 931 932 933 934 935
      break;
   case GLSL_TYPE_UINT:
      return nir_type_uint32;
      break;
   case GLSL_TYPE_INT:
      return nir_type_int32;
      break;
936 937 938 939 940 941
   case GLSL_TYPE_UINT16:
      return nir_type_uint16;
      break;
   case GLSL_TYPE_INT16:
      return nir_type_int16;
      break;
Karol Herbst's avatar
Karol Herbst committed
942 943 944 945
   case GLSL_TYPE_UINT8:
      return nir_type_uint8;
   case GLSL_TYPE_INT8:
      return nir_type_int8;
946 947 948 949 950 951
   case GLSL_TYPE_UINT64:
      return nir_type_uint64;
      break;
   case GLSL_TYPE_INT64:
      return nir_type_int64;
      break;
952 953 954
   case GLSL_TYPE_FLOAT:
      return nir_type_float32;
      break;
955 956 957
   case GLSL_TYPE_FLOAT16:
      return nir_type_float16;
      break;
958 959 960
   case GLSL_TYPE_DOUBLE:
      return nir_type_float64;
      break;
961 962 963 964 965 966 967 968 969 970 971 972

   case GLSL_TYPE_SAMPLER:
   case GLSL_TYPE_IMAGE:
   case GLSL_TYPE_ATOMIC_UINT:
   case GLSL_TYPE_STRUCT:
   case GLSL_TYPE_INTERFACE:
   case GLSL_TYPE_ARRAY:
   case GLSL_TYPE_VOID:
   case GLSL_TYPE_SUBROUTINE:
   case GLSL_TYPE_FUNCTION:
   case GLSL_TYPE_ERROR:
      return nir_type_invalid;
973
   }
974 975

   unreachable("unknown type");
976 977
}

978 979 980 981 982 983
static inline nir_alu_type
nir_get_nir_type_for_glsl_type(const struct glsl_type *type)
{
   return nir_get_nir_type_for_glsl_base_type(glsl_get_base_type(type));
}

984 985
nir_op nir_type_conversion_op(nir_alu_type src, nir_alu_type dst,
                              nir_rounding_mode rnd);
986

Karol Herbst's avatar
Karol Herbst committed
987 988 989 990
static inline nir_op
nir_op_vec(unsigned components)
{
   switch (components) {
991
   case  1: return nir_op_mov;
Karol Herbst's avatar
Karol Herbst committed
992 993 994 995 996 997 998
   case  2: return nir_op_vec2;
   case  3: return nir_op_vec3;
   case  4: return nir_op_vec4;
   default: unreachable("bad component count");
   }
}

999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079
static inline bool
nir_is_float_control_signed_zero_inf_nan_preserve(unsigned execution_mode, unsigned bit_size)
{
    return (16 == bit_size && execution_mode & FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP16) ||
        (32 == bit_size && execution_mode & FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP32) ||
        (64 == bit_size && execution_mode & FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP64);
}

static inline bool
nir_is_denorm_flush_to_zero(unsigned execution_mode, unsigned bit_size)
{
    return (16 == bit_size && execution_mode & FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP16) ||
        (32 == bit_size && execution_mode & FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP32) ||
        (64 == bit_size && execution_mode & FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP64);
}

static inline bool
nir_is_denorm_preserve(unsigned execution_mode, unsigned bit_size)
{
    return (16 == bit_size && execution_mode & FLOAT_CONTROLS_DENORM_PRESERVE_FP16) ||
        (32 == bit_size && execution_mode & FLOAT_CONTROLS_DENORM_PRESERVE_FP32) ||
        (64 == bit_size && execution_mode & FLOAT_CONTROLS_DENORM_PRESERVE_FP64);
}

static inline bool
nir_is_rounding_mode_rtne(unsigned execution_mode, unsigned bit_size)
{
    return (16 == bit_size && execution_mode & FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP16) ||
        (32 == bit_size && execution_mode & FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP32) ||
        (64 == bit_size && execution_mode & FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP64);
}

static inline bool
nir_is_rounding_mode_rtz(unsigned execution_mode, unsigned bit_size)
{
    return (16 == bit_size && execution_mode & FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16) ||
        (32 == bit_size && execution_mode & FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32) ||
        (64 == bit_size && execution_mode & FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64);
}

static inline bool
nir_has_any_rounding_mode_rtz(unsigned execution_mode)
{
    return (execution_mode & FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16) ||
        (execution_mode & FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32) ||
        (execution_mode & FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64);
}

static inline bool
nir_has_any_rounding_mode_rtne(unsigned execution_mode)
{
    return (execution_mode & FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP16) ||
        (execution_mode & FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP32) ||
        (execution_mode & FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP64);
}

static inline nir_rounding_mode
nir_get_rounding_mode_from_float_controls(unsigned execution_mode,
                                          nir_alu_type type)
{
   if (nir_alu_type_get_base_type(type) != nir_type_float)
      return nir_rounding_mode_undef;

   unsigned bit_size = nir_alu_type_get_type_size(type);

   if (nir_is_rounding_mode_rtz(execution_mode, bit_size))
      return nir_rounding_mode_rtz;
   if (nir_is_rounding_mode_rtne(execution_mode, bit_size))
      return nir_rounding_mode_rtne;
   return nir_rounding_mode_undef;
}

static inline bool
nir_has_any_rounding_mode_enabled(unsigned execution_mode)
{
   bool result =
      nir_has_any_rounding_mode_rtne(execution_mode) ||
      nir_has_any_rounding_mode_rtz(execution_mode);
   return result;
}

1080
typedef enum {
1081 1082 1083 1084 1085 1086 1087 1088
   /**
    * Operation where the first two sources are commutative.
    *
    * For 2-source operations, this just mathematical commutativity.  Some
    * 3-source operations, like ffma, are only commutative in the first two
    * sources.
    */
   NIR_OP_IS_2SRC_COMMUTATIVE = (1 << 0),
1089 1090 1091
   NIR_OP_IS_ASSOCIATIVE = (1 << 1),
} nir_op_algebraic_property;

1092 1093 1094 1095 1096 1097
typedef struct {
   const char *name;

   unsigned num_inputs;

   /**
1098
    * The number of components in the output
1099
    *
1100 1101 1102 1103
    * If non-zero, this is the size of the output and input sizes are
    * explicitly given; swizzle and writemask are still in effect, but if
    * the output component is masked out, then the input component may
    * still be in use.
1104
    *
1105 1106 1107 1108 1109 1110 1111 1112
    * If zero, the opcode acts in the standard, per-component manner; the
    * operation is performed on each component (except the ones that are
    * masked out) with the input being taken from the input swizzle for
    * that component.
    *
    * The size of some of the inputs may be given (i.e. non-zero) even
    * though output_size is zero; in that case, the inputs with a zero
    * size act per-component, while the inputs with non-zero size don't.
1113 1114 1115 1116
    */
   unsigned output_size;

   /**
Jason Ekstrand's avatar
Jason Ekstrand committed
1117 1118
    * The type of vector that the instruction outputs. Note that the
    * staurate modifier is only allowed on outputs with the float type.
1119 1120 1121 1122 1123
    */

   nir_alu_type output_type;

   /**
1124
    * The number of components in each input
1125
    */
1126
   unsigned input_sizes[NIR_MAX_VEC_COMPONENTS];
1127 1128

   /**
Jason Ekstrand's avatar
Jason Ekstrand committed
1129 1130 1131
    * The type of vector that each input takes. Note that negate and
    * absolute value are only allowed on inputs with int or float type and
    * behave differently on the two.
1132
    */
1133
   nir_alu_type input_types[NIR_MAX_VEC_COMPONENTS];
1134 1135

   nir_op_algebraic_property algebraic_properties;
1136 1137 1138

   /* Whether this represents a numeric conversion opcode */
   bool is_conversion;
1139 1140 1141 1142 1143 1144 1145
} nir_op_info;

extern const nir_op_info nir_op_infos[nir_num_opcodes];

typedef struct nir_alu_instr {
   nir_instr instr;
   nir_op op;
1146 1147 1148 1149 1150 1151 1152 1153 1154

   /** Indicates that this ALU instruction generates an exact value
    *
    * This is kind of a mixture of GLSL "precise" and "invariant" and not
    * really equivalent to either.  This indicates that the value generated by
    * this operation is high-precision and any code transformations that touch
    * it must ensure that the resulting value is bit-for-bit identical to the
    * original.
    */
1155 1156 1157 1158 1159 1160 1161 1162
   bool exact:1;

   /**
    * Indicates that this instruction do not cause wrapping to occur, in the
    * form of overflow or underflow.
    */
   bool no_signed_wrap:1;
   bool no_unsigned_wrap:1;
1163

1164 1165 1166 1167
   nir_alu_dest dest;
   nir_alu_src src[];
} nir_alu_instr;

1168 1169 1170 1171 1172
void nir_alu_src_copy(nir_alu_src *dest, const nir_alu_src *src,
                      nir_alu_instr *instr);
void nir_alu_dest_copy(nir_alu_dest *dest, const nir_alu_dest *src,
                       nir_alu_instr *instr);

1173 1174
/* is this source channel used? */
static inline bool
1175 1176
nir_alu_instr_channel_used(const nir_alu_instr *instr, unsigned src,
                           unsigned channel)
1177 1178 1179 1180 1181 1182 1183
{
   if (nir_op_infos[instr->op].input_sizes[src] > 0)
      return channel < nir_op_infos[instr->op].input_sizes[src];

   return (instr->dest.write_mask >> channel) & 1;
}

1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196
static inline nir_component_mask_t
nir_alu_instr_src_read_mask(const nir_alu_instr *instr, unsigned src)
{
   nir_component_mask_t read_mask = 0;
   for (unsigned c = 0; c < NIR_MAX_VEC_COMPONENTS; c++) {
      if (!nir_alu_instr_channel_used(instr, src, c))
         continue;

      read_mask |= (1 << instr->src[src].swizzle[c]);
   }
   return read_mask;
}

1197 1198
/**
 * Get the number of channels used for a source
1199 1200
 */
static inline unsigned
1201
nir_ssa_alu_instr_src_components(const nir_alu_instr *instr, unsigned src)
1202 1203 1204 1205
{
   if (nir_op_infos[instr->op].input_sizes[src] > 0)
      return nir_op_infos[instr->op].input_sizes[src];

1206
   return nir_dest_num_components(instr->dest.dest);
1207 1208
}

1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231
static inline bool
nir_alu_instr_is_comparison(const nir_alu_instr *instr)
{
   switch (instr->op) {
   case nir_op_flt:
   case nir_op_fge:
   case nir_op_feq:
   case nir_op_fne:
   case nir_op_ilt:
   case nir_op_ult:
   case nir_op_ige:
   case nir_op_uge:
   case nir_op_ieq:
   case nir_op_ine:
   case nir_op_i2b1:
   case nir_op_f2b1:
   case nir_op_inot:
      return true;
   default:
      return false;
   }
}

1232
bool nir_const_value_negative_equal(nir_const_value c1, nir_const_value c2,
1233
                                    nir_alu_type full_type);
1234

1235 1236 1237
bool nir_alu_srcs_equal(const nir_alu_instr *alu1, const nir_alu_instr *alu2,
                        unsigned src1, unsigned src2);

1238 1239 1240 1241
bool nir_alu_srcs_negative_equal(const nir_alu_instr *alu1,
                                 const nir_alu_instr *alu2,
                                 unsigned src1, unsigned src2);

1242 1243 1244
typedef enum {
   nir_deref_type_var,
   nir_deref_type_array,
1245
   nir_deref_type_array_wildcard,
1246
   nir_deref_type_ptr_as_array,
1247 1248
   nir_deref_type_struct,
   nir_deref_type_cast,
1249 1250
} nir_deref_type;

1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279
typedef struct {
   nir_instr instr;

   /** The type of this deref instruction */
   nir_deref_type deref_type;

   /** The mode of the underlying variable */
   nir_variable_mode mode;

   /** The dereferenced type of the resulting pointer value */
   const struct glsl_type *type;

   union {
      /** Variable being dereferenced if deref_type is a deref_var */
      nir_variable *var;

      /** Parent deref if deref_type is not deref_var */
      nir_src parent;
   };

   /** Additional deref parameters */
   union {
      struct {
         nir_src index;
      } arr;

      struct {
         unsigned index;
      } strct;
1280 1281 1282 1283

      struct {
         unsigned ptr_stride;
      } cast;
1284 1285 1286 1287 1288 1289
   };

   /** Destination to store the resulting "pointer" */
   nir_dest dest;
} nir_deref_instr;

1290
static inline nir_deref_instr *nir_src_as_deref(nir_src src);
1291

1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313
static inline nir_deref_instr *
nir_deref_instr_parent(const nir_deref_instr *instr)
{
   if (instr->deref_type == nir_deref_type_var)
      return NULL;
   else
      return nir_src_as_deref(instr->parent);
}

static inline nir_variable *
nir_deref_instr_get_variable(const nir_deref_instr *instr)
{
   while (instr->deref_type != nir_deref_type_var) {
      if (instr->deref_type == nir_deref_type_cast)
         return NULL;

      instr = nir_deref_instr_parent(instr);
   }

   return instr->var;
}

1314
bool nir_deref_instr_has_indirect(nir_deref_instr *instr);
1315
bool nir_deref_instr_is_known_out_of_bounds(nir_deref_instr *instr);
1316
bool nir_deref_instr_has_complex_use(nir_deref_instr *instr);
1317

1318 1319
bool nir_deref_instr_remove_if_unused(nir_deref_instr *instr);

1320 1321
unsigned nir_deref_instr_ptr_as_array_stride(nir_deref_instr *instr);

1322 1323 1324
typedef struct {
   nir_instr instr;

1325
   struct nir_function *callee;
1326 1327 1328

   unsigned num_params;
   nir_src params[];
1329 1330 1331 1332
} nir_call_instr;

#include "nir_intrinsics.h"

1333
#define NIR_INTRINSIC_MAX_CONST_INDEX 4
Rob Clark's avatar
Rob Clark committed
1334

Jason Ekstrand's avatar
Jason Ekstrand committed
1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360
/** Represents an intrinsic
 *
 * An intrinsic is an instruction type for handling things that are
 * more-or-less regular operations but don't just consume and produce SSA
 * values like ALU operations do.  Intrinsics are not for things that have
 * special semantic meaning such as phi nodes and parallel copies.
 * Examples of intrinsics include variable load/store operations, system
 * value loads, and the like.  Even though texturing more-or-less falls
 * under this category, texturing is its own instruction type because
 * trying to represent texturing with intrinsics would lead to a
 * combinatorial explosion of intrinsic opcodes.
 *
 * By having a single instruction type for handling a lot of different
 * cases, optimization passes can look for intrinsics and, for the most
 * part, completely ignore them.  Each intrinsic type also has a few
 * possible flags that govern whether or not they can be reordered or
 * eliminated.  That way passes like dead code elimination can still work
 * on intrisics without understanding the meaning of each.
 *
 * Each intrinsic has some number of constant indices, some number of
 * variables, and some number of sources.  What these sources, variables,
 * and indices mean depends on the intrinsic and is documented with the
 * intrinsic declaration in nir_intrinsics.h.  Intrinsics and texture
 * instructions are the only types of instruction that can operate on
 * variables.
 */
1361 1362 1363 1364 1365 1366 1367
typedef struct {
   nir_instr instr;

   nir_intrinsic_op intrinsic;

   nir_dest dest;

Jason Ekstrand's avatar
Jason Ekstrand committed
1368 1369 1370 1371 1372 1373 1374 1375
   /** number of components if this is a vectorized intrinsic
    *
    * Similarly to ALU operations, some intrinsics are vectorized.
    * An intrinsic is vectorized if nir_intrinsic_infos.dest_components == 0.
    * For vectorized intrinsics, the num_components field specifies the
    * number of destination components and the number of source components
    * for all sources with nir_intrinsic_infos.src_components[i] == 0.
    */
Jason Ekstrand's avatar
Jason Ekstrand committed
1376 1377
   uint8_t num_components;

Rob Clark's avatar
Rob Clark committed
1378
   int const_index[NIR_INTRINSIC_MAX_CONST_INDEX];
1379 1380 1381 1382

   nir_src src[];
} nir_intrinsic_instr;

1383 1384 1385 1386 1387 1388
static inline nir_variable *
nir_intrinsic_get_var(nir_intrinsic_instr *intrin, unsigned i)
{
   return nir_deref_instr_get_variable(nir_src_as_deref(intrin->src[i]));
}

1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406
typedef enum {
   /* Memory ordering. */
   NIR_MEMORY_ACQUIRE        = 1 << 0,
   NIR_MEMORY_RELEASE        = 1 << 1,

   /* Memory visibility operations. */
   NIR_MEMORY_MAKE_AVAILABLE = 1 << 3,
   NIR_MEMORY_MAKE_VISIBLE   = 1 << 4,
} nir_memory_semantics;

typedef enum {
   NIR_SCOPE_DEVICE,
   NIR_SCOPE_QUEUE_FAMILY,
   NIR_SCOPE_WORKGROUP,
   NIR_SCOPE_SUBGROUP,
   NIR_SCOPE_INVOCATION,
} nir_scope;

1407 1408 1409 1410 1411 1412 1413
/**
 * \name NIR intrinsics semantic flags
 *
 * information about what the compiler can do with the intrinsics.
 *
 * \sa nir_intrinsic_info::flags
 */
1414 1415 1416 1417 1418 1419
typedef enum {
   /**
    * whether the intrinsic can be safely eliminated if none of its output
    * value is not being used.
    */
   NIR_INTRINSIC_CAN_ELIMINATE = (1 << 0),
1420

1421 1422 1423 1424 1425 1426 1427
   /**
    * Whether the intrinsic can be reordered with respect to any other
    * intrinsic, i.e. whether the only reordering dependencies of the
    * intrinsic are due to the register reads/writes.
    */
   NIR_INTRINSIC_CAN_REORDER = (1 << 1),
} nir_intrinsic_semantic_flag;
1428

Rob Clark's avatar
Rob Clark committed
1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445
/**
 * \name NIR intrinsics const-index flag
 *
 * Indicates the usage of a const_index slot.
 *
 * \sa nir_intrinsic_info::index_map
 */
typedef enum {
   /**
    * Generally instructions that take a offset src argument, can encode
    * a constant 'base' value which is added to the offset.
    */
   NIR_INTRINSIC_BASE = 1,

   /**
    * For store instructions, a writemask for the store.
    */
1446
   NIR_INTRINSIC_WRMASK,
Rob Clark's avatar
Rob Clark committed
1447 1448 1449 1450

   /**
    * The stream-id for GS emit_vertex/end_primitive intrinsics.
    */
1451
   NIR_INTRINSIC_STREAM_ID,
Rob Clark's avatar
Rob Clark committed
1452 1453 1454 1455

   /**
    * The clip-plane id for load_user_clip_plane intrinsic.
    */
1456
   NIR_INTRINSIC_UCP_ID,
Rob Clark's avatar
Rob Clark committed
1457

1458 1459 1460 1461
   /**
    * The amount of data, starting from BASE, that this instruction may
    * access.  This is used to provide bounds if the offset is not constant.
    */
1462
   NIR_INTRINSIC_RANGE,
1463

1464 1465 1466
   /**
    * The Vulkan descriptor set for vulkan_resource_index intrinsic.
    */
1467
   NIR_INTRINSIC_DESC_SET,
1468 1469 1470 1471

   /**
    * The Vulkan descriptor set binding for vulkan_resource_index intrinsic.
    */
1472
   NIR_INTRINSIC_BINDING,
1473

1474 1475 1476
   /**
    * Component offset.
    */
1477
   NIR_INTRINSIC_COMPONENT,
1478

1479 1480 1481
   /**
    * Interpolation mode (only meaningful for FS inputs).
    */
1482
   NIR_INTRINSIC_INTERP_MODE,
1483

1484 1485 1486
   /**
    * A binary nir_op to use when performing a reduction or scan operation
    */
1487
   NIR_INTRINSIC_REDUCTION_OP,
1488 1489 1490 1491

   /**
    * Cluster size for reduction operations
    */
1492
   NIR_INTRINSIC_CLUSTER_SIZE,
1493

1494 1495 1496
   /**
    * Parameter index for a load_param intrinsic
    */
1497
   NIR_INTRINSIC_PARAM_IDX,
1498

1499 1500 1501 1502 1503
   /**
    * Image dimensionality for image intrinsics
    *
    * One of GLSL_SAMPLER_DIM_*
    */
1504
   NIR_INTRINSIC_IMAGE_DIM,
1505 1506 1507 1508

   /**
    * Non-zero if we are accessing an array image
    */
1509
   NIR_INTRINSIC_IMAGE_ARRAY,
1510 1511 1512 1513

   /**
    * Image format for image intrinsics
    */
1514
   NIR_INTRINSIC_FORMAT,
1515 1516

   /**
1517
    * Access qualifiers for image and memory access intrinsics
1518
    */
1519
   NIR_INTRINSIC_ACCESS,
1520

1521 1522 1523 1524 1525 1526 1527 1528 1529
   /**
    * Alignment for offsets and addresses
    *
    * These two parameters, specify an alignment in terms of a multiplier and
    * an offset.  The offset or address parameter X of the intrinsic is
    * guaranteed to satisfy the following:
    *
    *                (X - align_offset) % align_mul == 0
    */
1530 1531
   NIR_INTRINSIC_ALIGN_MUL,
   NIR_INTRINSIC_ALIGN_OFFSET,
1532

1533 1534 1535
   /**
    * The Vulkan descriptor type for a vulkan_resource_[re]index intrinsic.
    */
1536
   NIR_INTRINSIC_DESC_TYPE,
1537

1538 1539 1540
   /**
    * The nir_alu_type of a uniform/input/output
    */
1541
   NIR_INTRINSIC_TYPE,
1542

1543 1544 1545 1546
   /**
    * The swizzle mask for the instructions
    * SwizzleInvocationsAMD and SwizzleInvocationsMaskedAMD
    */
1547
   NIR_INTRINSIC_SWIZZLE_MASK,
1548

1549
   /* Separate source/dest access flags for copies */
1550 1551
   NIR_INTRINSIC_SRC_ACCESS,
   NIR_INTRINSIC_DST_ACCESS,
1552

1553 1554 1555
   /* Driver location for nir_load_patch_location_ir3 */
   NIR_INTRINSIC_DRIVER_LOCATION,

1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570
   /**
    * Mask of nir_memory_semantics, includes ordering and visibility.
    */
   NIR_INTRINSIC_MEMORY_SEMANTICS,

   /**
    * Mask of nir_variable_modes affected by the memory operation.
    */
   NIR_INTRINSIC_MEMORY_MODES,</