draw_llvm.c 161 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
/**************************************************************************
 *
 * Copyright 2010 VMware, Inc.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 **************************************************************************/

28 29 30 31
#include "draw_llvm.h"

#include "draw_context.h"
#include "draw_vs.h"
32
#include "draw_gs.h"
33 34

#include "gallivm/lp_bld_arit.h"
35
#include "gallivm/lp_bld_arit_overflow.h"
36 37
#include "gallivm/lp_bld_bitarit.h"
#include "gallivm/lp_bld_gather.h"
38 39
#include "gallivm/lp_bld_logic.h"
#include "gallivm/lp_bld_const.h"
Dave Airlie's avatar
Dave Airlie committed
40
#include "gallivm/lp_bld_coro.h"
41
#include "gallivm/lp_bld_swizzle.h"
42 43 44 45 46
#include "gallivm/lp_bld_struct.h"
#include "gallivm/lp_bld_type.h"
#include "gallivm/lp_bld_flow.h"
#include "gallivm/lp_bld_debug.h"
#include "gallivm/lp_bld_tgsi.h"
47
#include "gallivm/lp_bld_nir.h"
48
#include "gallivm/lp_bld_printf.h"
49
#include "gallivm/lp_bld_intr.h"
50
#include "gallivm/lp_bld_init.h"
51
#include "gallivm/lp_bld_type.h"
52 53
#include "gallivm/lp_bld_pack.h"
#include "gallivm/lp_bld_format.h"
54
#include "gallivm/lp_bld_misc.h"
55
#include "tgsi/tgsi_exec.h"
Jose Fonseca's avatar
Jose Fonseca committed
56
#include "tgsi/tgsi_dump.h"
57

58
#include "util/u_math.h"
59
#include "util/u_pointer.h"
60
#include "util/u_string.h"
61
#include "util/simple_list.h"
62 63
#include "nir_serialize.h"
#include "util/mesa-sha1.h"
64 65
#define DEBUG_STORE 0

66

67
static void
68
draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var);
69

70

Zack Rusin's avatar
Zack Rusin committed
71
struct draw_gs_llvm_iface {
72
   struct lp_build_gs_iface base;
Zack Rusin's avatar
Zack Rusin committed
73 74 75 76 77

   struct draw_gs_llvm_variant *variant;
   LLVMValueRef input;
};

78
static inline const struct draw_gs_llvm_iface *
79
draw_gs_llvm_iface(const struct lp_build_gs_iface *iface)
Zack Rusin's avatar
Zack Rusin committed
80 81 82 83
{
   return (const struct draw_gs_llvm_iface *)iface;
}

Dave Airlie's avatar
Dave Airlie committed
84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110
struct draw_tcs_llvm_iface {
   struct lp_build_tcs_iface base;

   struct draw_tcs_llvm_variant *variant;
   LLVMValueRef input;
   LLVMValueRef output;
};

static inline const struct draw_tcs_llvm_iface *
draw_tcs_llvm_iface(const struct lp_build_tcs_iface *iface)
{
   return (const struct draw_tcs_llvm_iface *)iface;
}

struct draw_tes_llvm_iface {
   struct lp_build_tes_iface base;

   struct draw_tes_llvm_variant *variant;
   LLVMValueRef input;
};

static inline const struct draw_tes_llvm_iface *
draw_tes_llvm_iface(const struct lp_build_tes_iface *iface)
{
   return (const struct draw_tes_llvm_iface *)iface;
}

111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
/**
 * Create LLVM type for draw_vertex_buffer.
 */
static LLVMTypeRef
create_jit_dvbuffer_type(struct gallivm_state *gallivm,
                         const char *struct_name)
{
   LLVMTargetDataRef target = gallivm->target;
   LLVMTypeRef dvbuffer_type;
   LLVMTypeRef elem_types[DRAW_JIT_DVBUFFER_NUM_FIELDS];
   LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context);

   elem_types[DRAW_JIT_DVBUFFER_MAP] =
      LLVMPointerType(LLVMIntTypeInContext(gallivm->context, 8), 0);
   elem_types[DRAW_JIT_DVBUFFER_SIZE] = int32_type;

   dvbuffer_type = LLVMStructTypeInContext(gallivm->context, elem_types,
Brian Paul's avatar
Brian Paul committed
128
                                           ARRAY_SIZE(elem_types), 0);
129

130
   (void) target; /* silence unused var warning for non-debug build */
131 132 133 134 135 136 137 138 139 140
   LP_CHECK_MEMBER_OFFSET(struct draw_vertex_buffer, map,
                          target, dvbuffer_type,
                          DRAW_JIT_DVBUFFER_MAP);
   LP_CHECK_MEMBER_OFFSET(struct draw_vertex_buffer, size,
                          target, dvbuffer_type,
                          DRAW_JIT_DVBUFFER_SIZE);

   return dvbuffer_type;
}

141 142 143 144
/**
 * Create LLVM type for struct draw_jit_texture
 */
static LLVMTypeRef
145
create_jit_texture_type(struct gallivm_state *gallivm, const char *struct_name)
146
{
147
   LLVMTargetDataRef target = gallivm->target;
148
   LLVMTypeRef texture_type;
149 150 151 152 153 154
   LLVMTypeRef elem_types[DRAW_JIT_TEXTURE_NUM_FIELDS];
   LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context);

   elem_types[DRAW_JIT_TEXTURE_WIDTH]  =
   elem_types[DRAW_JIT_TEXTURE_HEIGHT] =
   elem_types[DRAW_JIT_TEXTURE_DEPTH] =
155 156
   elem_types[DRAW_JIT_TEXTURE_NUM_SAMPLES] =
   elem_types[DRAW_JIT_TEXTURE_SAMPLE_STRIDE] =
157
   elem_types[DRAW_JIT_TEXTURE_FIRST_LEVEL] =
158
   elem_types[DRAW_JIT_TEXTURE_LAST_LEVEL] = int32_type;
159 160
   elem_types[DRAW_JIT_TEXTURE_BASE] =
      LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0);
161 162
   elem_types[DRAW_JIT_TEXTURE_ROW_STRIDE] =
   elem_types[DRAW_JIT_TEXTURE_IMG_STRIDE] =
163
   elem_types[DRAW_JIT_TEXTURE_MIP_OFFSETS] =
164 165 166
      LLVMArrayType(int32_type, PIPE_MAX_TEXTURE_LEVELS);

   texture_type = LLVMStructTypeInContext(gallivm->context, elem_types,
Brian Paul's avatar
Brian Paul committed
167
                                          ARRAY_SIZE(elem_types), 0);
168

169
   (void) target; /* silence unused var warning for non-debug build */
170 171 172 173 174 175 176 177 178
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width,
                          target, texture_type,
                          DRAW_JIT_TEXTURE_WIDTH);
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height,
                          target, texture_type,
                          DRAW_JIT_TEXTURE_HEIGHT);
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, depth,
                          target, texture_type,
                          DRAW_JIT_TEXTURE_DEPTH);
179 180 181
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, base,
                          target, texture_type,
                          DRAW_JIT_TEXTURE_BASE);
182 183 184 185 186 187
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, row_stride,
                          target, texture_type,
                          DRAW_JIT_TEXTURE_ROW_STRIDE);
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, img_stride,
                          target, texture_type,
                          DRAW_JIT_TEXTURE_IMG_STRIDE);
188 189 190 191 192 193
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, first_level,
                          target, texture_type,
                          DRAW_JIT_TEXTURE_FIRST_LEVEL);
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, last_level,
                          target, texture_type,
                          DRAW_JIT_TEXTURE_LAST_LEVEL);
194
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, mip_offsets,
195
                          target, texture_type,
196
                          DRAW_JIT_TEXTURE_MIP_OFFSETS);
197 198 199 200 201 202
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, num_samples,
                          target, texture_type,
                          DRAW_JIT_TEXTURE_NUM_SAMPLES);
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, sample_stride,
                          target, texture_type,
                          DRAW_JIT_TEXTURE_SAMPLE_STRIDE);
203 204 205 206 207

   LP_CHECK_STRUCT_SIZE(struct draw_jit_texture, target, texture_type);

   return texture_type;
}
208 209


210
/**
211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226
 * Create LLVM type for struct draw_jit_sampler
 */
static LLVMTypeRef
create_jit_sampler_type(struct gallivm_state *gallivm, const char *struct_name)
{
   LLVMTargetDataRef target = gallivm->target;
   LLVMTypeRef sampler_type;
   LLVMTypeRef elem_types[DRAW_JIT_SAMPLER_NUM_FIELDS];

   elem_types[DRAW_JIT_SAMPLER_MIN_LOD] =
   elem_types[DRAW_JIT_SAMPLER_MAX_LOD] =
   elem_types[DRAW_JIT_SAMPLER_LOD_BIAS] = LLVMFloatTypeInContext(gallivm->context);
   elem_types[DRAW_JIT_SAMPLER_BORDER_COLOR] =
      LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4);

   sampler_type = LLVMStructTypeInContext(gallivm->context, elem_types,
Brian Paul's avatar
Brian Paul committed
227
                                          ARRAY_SIZE(elem_types), 0);
228

229
   (void) target; /* silence unused var warning for non-debug build */
230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_sampler, min_lod,
                          target, sampler_type,
                          DRAW_JIT_SAMPLER_MIN_LOD);
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_sampler, max_lod,
                          target, sampler_type,
                          DRAW_JIT_SAMPLER_MAX_LOD);
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_sampler, lod_bias,
                          target, sampler_type,
                          DRAW_JIT_SAMPLER_LOD_BIAS);
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_sampler, border_color,
                          target, sampler_type,
                          DRAW_JIT_SAMPLER_BORDER_COLOR);

   LP_CHECK_STRUCT_SIZE(struct draw_jit_sampler, target, sampler_type);

   return sampler_type;
}

248 249 250 251 252 253 254 255 256 257 258 259 260 261 262
/**
 * Create LLVM type for struct draw_jit_texture
 */
static LLVMTypeRef
create_jit_image_type(struct gallivm_state *gallivm, const char *struct_name)
{
   LLVMTargetDataRef target = gallivm->target;
   LLVMTypeRef image_type;
   LLVMTypeRef elem_types[DRAW_JIT_IMAGE_NUM_FIELDS];
   LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context);

   elem_types[DRAW_JIT_IMAGE_WIDTH]  =
   elem_types[DRAW_JIT_IMAGE_HEIGHT] =
   elem_types[DRAW_JIT_IMAGE_DEPTH] =
   elem_types[DRAW_JIT_IMAGE_ROW_STRIDE] =
263 264 265
   elem_types[DRAW_JIT_IMAGE_IMG_STRIDE] =
   elem_types[DRAW_JIT_IMAGE_NUM_SAMPLES] =
   elem_types[DRAW_JIT_IMAGE_SAMPLE_STRIDE] = int32_type;
266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290
   elem_types[DRAW_JIT_IMAGE_BASE] =
      LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0);

   image_type = LLVMStructTypeInContext(gallivm->context, elem_types,
                                          ARRAY_SIZE(elem_types), 0);

   (void) target; /* silence unused var warning for non-debug build */
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_image, width,
                          target, image_type,
                          DRAW_JIT_IMAGE_WIDTH);
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_image, height,
                          target, image_type,
                          DRAW_JIT_IMAGE_HEIGHT);
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_image, depth,
                          target, image_type,
                          DRAW_JIT_IMAGE_DEPTH);
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_image, base,
                          target, image_type,
                          DRAW_JIT_IMAGE_BASE);
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_image, row_stride,
                          target, image_type,
                          DRAW_JIT_IMAGE_ROW_STRIDE);
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_image, img_stride,
                          target, image_type,
                          DRAW_JIT_IMAGE_IMG_STRIDE);
291 292 293 294 295 296
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_image, num_samples,
                          target, image_type,
                          DRAW_JIT_IMAGE_NUM_SAMPLES);
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_image, sample_stride,
                          target, image_type,
                          DRAW_JIT_IMAGE_SAMPLE_STRIDE);
297 298 299 300 301

   LP_CHECK_STRUCT_SIZE(struct draw_jit_image, target, image_type);

   return image_type;
}
302 303 304

/**
 * Create LLVM type for struct draw_jit_context
305 306 307
 */
static LLVMTypeRef
create_jit_context_type(struct gallivm_state *gallivm,
308
                        LLVMTypeRef texture_type, LLVMTypeRef sampler_type,
309
                        LLVMTypeRef image_type,
310
                        const char *struct_name)
311 312 313
{
   LLVMTargetDataRef target = gallivm->target;
   LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context);
314
   LLVMTypeRef int_type = LLVMInt32TypeInContext(gallivm->context);
315
   LLVMTypeRef elem_types[DRAW_JIT_CTX_NUM_FIELDS];
316 317
   LLVMTypeRef context_type;

318 319
   elem_types[0] = LLVMArrayType(LLVMPointerType(float_type, 0), /* vs_constants */
                                 LP_MAX_TGSI_CONST_BUFFERS);
320 321 322
   elem_types[1] = LLVMArrayType(int_type, /* num_vs_constants */
                                 LP_MAX_TGSI_CONST_BUFFERS);
   elem_types[2] = LLVMPointerType(LLVMArrayType(LLVMArrayType(float_type, 4),
323
                                                 DRAW_TOTAL_CLIP_PLANES), 0);
324
   elem_types[3] = LLVMPointerType(float_type, 0); /* viewports */
325
   elem_types[4] = LLVMArrayType(texture_type,
326
                                 PIPE_MAX_SHADER_SAMPLER_VIEWS); /* textures */
327
   elem_types[5] = LLVMArrayType(sampler_type,
328
                                 PIPE_MAX_SAMPLERS); /* samplers */
329 330 331
   elem_types[6] = LLVMArrayType(image_type,
                                 PIPE_MAX_SHADER_IMAGES); /* images */
   elem_types[7] = LLVMArrayType(LLVMPointerType(int_type, 0), /* vs_ssbo */
332
                                 LP_MAX_TGSI_SHADER_BUFFERS);
333
   elem_types[8] = LLVMArrayType(int_type, /* num_vs_ssbos */
334
                                 LP_MAX_TGSI_SHADER_BUFFERS);
335
   context_type = LLVMStructTypeInContext(gallivm->context, elem_types,
Brian Paul's avatar
Brian Paul committed
336
                                          ARRAY_SIZE(elem_types), 0);
337 338

   (void) target; /* silence unused var warning for non-debug build */
339
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants,
340
                          target, context_type, DRAW_JIT_CTX_CONSTANTS);
341 342
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, num_vs_constants,
                          target, context_type, DRAW_JIT_CTX_NUM_CONSTANTS);
343
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, planes,
344
                          target, context_type, DRAW_JIT_CTX_PLANES);
345
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, viewports,
346
                          target, context_type, DRAW_JIT_CTX_VIEWPORT);
347 348 349
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures,
                          target, context_type,
                          DRAW_JIT_CTX_TEXTURES);
350 351 352
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, samplers,
                          target, context_type,
                          DRAW_JIT_CTX_SAMPLERS);
353 354
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, images,
                          target, context_type, DRAW_JIT_CTX_IMAGES);
355 356 357 358
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_ssbos,
                          target, context_type, DRAW_JIT_CTX_SSBOS);
   LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, num_vs_ssbos,
                          target, context_type, DRAW_JIT_CTX_NUM_SSBOS);
359 360 361 362 363
   LP_CHECK_STRUCT_SIZE(struct draw_jit_context,
                        target, context_type);

   return context_type;
}
364

365

366 367 368 369 370 371 372
/**
 * Create LLVM type for struct draw_gs_jit_context
 */
static LLVMTypeRef
create_gs_jit_context_type(struct gallivm_state *gallivm,
                           unsigned vector_length,
                           LLVMTypeRef texture_type, LLVMTypeRef sampler_type,
373
                           LLVMTypeRef image_type,
374 375 376 377 378
                           const char *struct_name)
{
   LLVMTargetDataRef target = gallivm->target;
   LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context);
   LLVMTypeRef int_type = LLVMInt32TypeInContext(gallivm->context);
379
   LLVMTypeRef elem_types[DRAW_GS_JIT_CTX_NUM_FIELDS];
380 381 382 383
   LLVMTypeRef context_type;

   elem_types[0] = LLVMArrayType(LLVMPointerType(float_type, 0), /* constants */
                                 LP_MAX_TGSI_CONST_BUFFERS);
384 385 386
   elem_types[1] = LLVMArrayType(int_type, /* num_constants */
                                 LP_MAX_TGSI_CONST_BUFFERS);
   elem_types[2] = LLVMPointerType(LLVMArrayType(LLVMArrayType(float_type, 4),
387
                                                 DRAW_TOTAL_CLIP_PLANES), 0);
388
   elem_types[3] = LLVMPointerType(float_type, 0); /* viewports */
389

390
   elem_types[4] = LLVMArrayType(texture_type,
391
                                 PIPE_MAX_SHADER_SAMPLER_VIEWS); /* textures */
392
   elem_types[5] = LLVMArrayType(sampler_type,
393
                                 PIPE_MAX_SAMPLERS); /* samplers */
394 395 396
   elem_types[6] = LLVMArrayType(image_type,
                                 PIPE_MAX_SHADER_IMAGES); /* images */
   elem_types[7] = LLVMPointerType(LLVMPointerType(int_type, 0), 0);
397 398
   elem_types[8] = LLVMPointerType(LLVMVectorType(int_type,
                                                  vector_length), 0);
399 400
   elem_types[9] = LLVMPointerType(LLVMVectorType(int_type,
                                                  vector_length), 0);
401

402
   elem_types[10] = LLVMArrayType(LLVMPointerType(int_type, 0), /* ssbos */
403
                                 LP_MAX_TGSI_SHADER_BUFFERS);
404
   elem_types[11] = LLVMArrayType(int_type, /* num_ssbos */
405
                                 LP_MAX_TGSI_SHADER_BUFFERS);
406

407
   context_type = LLVMStructTypeInContext(gallivm->context, elem_types,
Brian Paul's avatar
Brian Paul committed
408
                                          ARRAY_SIZE(elem_types), 0);
409

410
   (void) target; /* silence unused var warning for non-debug build */
411
   LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, constants,
412
                          target, context_type, DRAW_GS_JIT_CTX_CONSTANTS);
413 414
   LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, num_constants,
                          target, context_type, DRAW_GS_JIT_CTX_NUM_CONSTANTS);
415
   LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, planes,
416
                          target, context_type, DRAW_GS_JIT_CTX_PLANES);
417
   LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, viewports,
418
                          target, context_type, DRAW_GS_JIT_CTX_VIEWPORT);
419 420 421 422 423 424
   LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, textures,
                          target, context_type,
                          DRAW_GS_JIT_CTX_TEXTURES);
   LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, samplers,
                          target, context_type,
                          DRAW_GS_JIT_CTX_SAMPLERS);
425 426 427 428 429 430 431 432 433
   LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, prim_lengths,
                          target, context_type,
                          DRAW_GS_JIT_CTX_PRIM_LENGTHS);
   LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, emitted_vertices,
                          target, context_type,
                          DRAW_GS_JIT_CTX_EMITTED_VERTICES);
   LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, emitted_prims,
                          target, context_type,
                          DRAW_GS_JIT_CTX_EMITTED_PRIMS);
434 435 436 437
   LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, ssbos,
                          target, context_type, DRAW_GS_JIT_CTX_SSBOS);
   LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, num_ssbos,
                          target, context_type, DRAW_GS_JIT_CTX_NUM_SSBOS);
438 439
   LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, images,
                          target, context_type, DRAW_GS_JIT_CTX_IMAGES);
440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460
   LP_CHECK_STRUCT_SIZE(struct draw_gs_jit_context,
                        target, context_type);

   return context_type;
}


static LLVMTypeRef
create_gs_jit_input_type(struct gallivm_state *gallivm)
{
   LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context);
   LLVMTypeRef input_array;

   input_array = LLVMVectorType(float_type, TGSI_NUM_CHANNELS); /* num primitives */
   input_array = LLVMArrayType(input_array, TGSI_NUM_CHANNELS); /* num channels */
   input_array = LLVMArrayType(input_array, PIPE_MAX_SHADER_INPUTS); /* num attrs per vertex */
   input_array = LLVMPointerType(input_array, 0); /* num vertices per prim */

   return input_array;
}

461 462 463 464
/**
 * Create LLVM type for struct pipe_vertex_buffer
 */
static LLVMTypeRef
465 466
create_jit_vertex_buffer_type(struct gallivm_state *gallivm,
                              const char *struct_name)
467 468
{
   LLVMTargetDataRef target = gallivm->target;
469
   LLVMTypeRef elem_types[4];
470
   LLVMTypeRef vb_type;
471

472 473 474
   elem_types[0] = LLVMInt16TypeInContext(gallivm->context);
   elem_types[1] = LLVMInt8TypeInContext(gallivm->context);
   elem_types[2] = LLVMInt32TypeInContext(gallivm->context);
475
   elem_types[3] = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0);
476

477
   vb_type = LLVMStructTypeInContext(gallivm->context, elem_types,
Brian Paul's avatar
Brian Paul committed
478
                                     ARRAY_SIZE(elem_types), 0);
479

480
   (void) target; /* silence unused var warning for non-debug build */
481 482
   LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, stride,
                          target, vb_type, 0);
483
   LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, is_user_buffer,
484
                          target, vb_type, 1);
485 486 487 488
   LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset,
                          target, vb_type, 2);
   LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer.resource,
                          target, vb_type, 3);
489 490 491 492

   LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer, target, vb_type);

   return vb_type;
493 494
}

495 496 497 498

/**
 * Create LLVM type for struct vertex_header;
 */
499
static LLVMTypeRef
500
create_jit_vertex_header(struct gallivm_state *gallivm, int data_elems)
501
{
502
   LLVMTargetDataRef target = gallivm->target;
503
   LLVMTypeRef elem_types[3];
504
   LLVMTypeRef vertex_header;
505 506
   char struct_name[24];

507
   snprintf(struct_name, 23, "vertex_header%d", data_elems);
508

509
   elem_types[DRAW_JIT_VERTEX_VERTEX_ID]  = LLVMIntTypeInContext(gallivm->context, 32);
510
   elem_types[DRAW_JIT_VERTEX_CLIP_POS]  = LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4);
511
   elem_types[DRAW_JIT_VERTEX_DATA]  = LLVMArrayType(elem_types[1], data_elems);
512

513
   vertex_header = LLVMStructTypeInContext(gallivm->context, elem_types,
Brian Paul's avatar
Brian Paul committed
514
                                           ARRAY_SIZE(elem_types), 0);
515 516 517

   /* these are bit-fields and we can't take address of them
      LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask,
518
      target, vertex_header,
519 520
      DRAW_JIT_VERTEX_CLIPMASK);
      LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag,
521
      target, vertex_header,
522 523
      DRAW_JIT_VERTEX_EDGEFLAG);
      LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad,
524
      target, vertex_header,
525 526
      DRAW_JIT_VERTEX_PAD);
      LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id,
527
      target, vertex_header,
528 529
      DRAW_JIT_VERTEX_VERTEX_ID);
   */
530 531
   (void) target; /* silence unused var warning for non-debug build */
   LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip_pos,
532
                          target, vertex_header,
533
                          DRAW_JIT_VERTEX_CLIP_POS);
534
   LP_CHECK_MEMBER_OFFSET(struct vertex_header, data,
535
                          target, vertex_header,
536 537
                          DRAW_JIT_VERTEX_DATA);

538 539 540
   assert(LLVMABISizeOfType(target, vertex_header) ==
          offsetof(struct vertex_header, data[data_elems]));

541
   return vertex_header;
542 543
}

544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703
/**
 * Create LLVM type for struct draw_tcs_jit_context
 */
static LLVMTypeRef
create_tcs_jit_context_type(struct gallivm_state *gallivm,
                            unsigned vector_length,
                            LLVMTypeRef texture_type, LLVMTypeRef sampler_type,
                            LLVMTypeRef image_type,
                            const char *struct_name)
{
   LLVMTargetDataRef target = gallivm->target;
   LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context);
   LLVMTypeRef int_type = LLVMInt32TypeInContext(gallivm->context);
   LLVMTypeRef elem_types[DRAW_TCS_JIT_CTX_NUM_FIELDS];
   LLVMTypeRef context_type;

   elem_types[0] = LLVMArrayType(LLVMPointerType(float_type, 0), /* constants */
                                 LP_MAX_TGSI_CONST_BUFFERS);
   elem_types[1] = LLVMArrayType(int_type, /* num_constants */
                                 LP_MAX_TGSI_CONST_BUFFERS);
   elem_types[2] = LLVMInt32TypeInContext(gallivm->context);
   elem_types[3] = LLVMInt32TypeInContext(gallivm->context);

   elem_types[4] = LLVMArrayType(texture_type,
                                 PIPE_MAX_SHADER_SAMPLER_VIEWS); /* textures */
   elem_types[5] = LLVMArrayType(sampler_type,
                                 PIPE_MAX_SAMPLERS); /* samplers */
   elem_types[6] = LLVMArrayType(image_type,
                                 PIPE_MAX_SHADER_IMAGES); /* images */

   elem_types[7] = LLVMArrayType(LLVMPointerType(int_type, 0), /* ssbos */
                                 LP_MAX_TGSI_SHADER_BUFFERS);
   elem_types[8] = LLVMArrayType(int_type, /* num_ssbos */
                                 LP_MAX_TGSI_SHADER_BUFFERS);

   context_type = LLVMStructTypeInContext(gallivm->context, elem_types,
                                          ARRAY_SIZE(elem_types), 0);

   (void) target; /* silence unused var warning for non-debug build */
   LP_CHECK_MEMBER_OFFSET(struct draw_tcs_jit_context, constants,
                          target, context_type, DRAW_TCS_JIT_CTX_CONSTANTS);
   LP_CHECK_MEMBER_OFFSET(struct draw_tcs_jit_context, num_constants,
                          target, context_type, DRAW_TCS_JIT_CTX_NUM_CONSTANTS);
   LP_CHECK_MEMBER_OFFSET(struct draw_tcs_jit_context, textures,
                          target, context_type,
                          DRAW_TCS_JIT_CTX_TEXTURES);
   LP_CHECK_MEMBER_OFFSET(struct draw_tcs_jit_context, samplers,
                          target, context_type,
                          DRAW_TCS_JIT_CTX_SAMPLERS);
   LP_CHECK_MEMBER_OFFSET(struct draw_tcs_jit_context, ssbos,
                          target, context_type, DRAW_TCS_JIT_CTX_SSBOS);
   LP_CHECK_MEMBER_OFFSET(struct draw_tcs_jit_context, num_ssbos,
                          target, context_type, DRAW_TCS_JIT_CTX_NUM_SSBOS);
   LP_CHECK_MEMBER_OFFSET(struct draw_tcs_jit_context, images,
                          target, context_type, DRAW_TCS_JIT_CTX_IMAGES);
   LP_CHECK_STRUCT_SIZE(struct draw_tcs_jit_context,
                        target, context_type);

   return context_type;
}

static LLVMTypeRef
create_tcs_jit_input_type(struct gallivm_state *gallivm)
{
   LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context);
   LLVMTypeRef input_array;

   input_array = LLVMArrayType(float_type, TGSI_NUM_CHANNELS); /* num channels */
   input_array = LLVMArrayType(input_array, NUM_TCS_INPUTS); /* num attrs per vertex */
   input_array = LLVMPointerType(input_array, 0); /* num vertices per prim */

   return input_array;
}

static LLVMTypeRef
create_tcs_jit_output_type(struct gallivm_state *gallivm)
{
   LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context);
   LLVMTypeRef output_array;

   output_array = LLVMArrayType(float_type, TGSI_NUM_CHANNELS); /* num channels */
   output_array = LLVMArrayType(output_array, PIPE_MAX_SHADER_INPUTS); /* num attrs per vertex */
   output_array = LLVMPointerType(output_array, 0); /* num vertices per prim */

   return output_array;
}

static LLVMTypeRef
create_tes_jit_input_type(struct gallivm_state *gallivm)
{
   LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context);
   LLVMTypeRef input_array;

   input_array = LLVMArrayType(float_type, TGSI_NUM_CHANNELS); /* num channels */
   input_array = LLVMArrayType(input_array, PIPE_MAX_SHADER_INPUTS); /* num attrs per vertex */
   input_array = LLVMPointerType(input_array, 0); /* num vertices per prim */

   return input_array;
}

/**
 * Create LLVM type for struct draw_tes_jit_context
 */
static LLVMTypeRef
create_tes_jit_context_type(struct gallivm_state *gallivm,
                            unsigned vector_length,
                            LLVMTypeRef texture_type, LLVMTypeRef sampler_type,
                            LLVMTypeRef image_type,
                            const char *struct_name)
{
   LLVMTargetDataRef target = gallivm->target;
   LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context);
   LLVMTypeRef int_type = LLVMInt32TypeInContext(gallivm->context);
   LLVMTypeRef elem_types[DRAW_TCS_JIT_CTX_NUM_FIELDS];
   LLVMTypeRef context_type;

   elem_types[0] = LLVMArrayType(LLVMPointerType(float_type, 0), /* constants */
                                 LP_MAX_TGSI_CONST_BUFFERS);
   elem_types[1] = LLVMArrayType(int_type, /* num_constants */
                                 LP_MAX_TGSI_CONST_BUFFERS);
   elem_types[2] = LLVMInt32TypeInContext(gallivm->context);
   elem_types[3] = LLVMInt32TypeInContext(gallivm->context);

   elem_types[4] = LLVMArrayType(texture_type,
                                 PIPE_MAX_SHADER_SAMPLER_VIEWS); /* textures */
   elem_types[5] = LLVMArrayType(sampler_type,
                                 PIPE_MAX_SAMPLERS); /* samplers */
   elem_types[6] = LLVMArrayType(image_type,
                                 PIPE_MAX_SHADER_IMAGES); /* images */

   elem_types[7] = LLVMArrayType(LLVMPointerType(int_type, 0), /* ssbos */
                                 LP_MAX_TGSI_SHADER_BUFFERS);
   elem_types[8] = LLVMArrayType(int_type, /* num_ssbos */
                                 LP_MAX_TGSI_SHADER_BUFFERS);

   context_type = LLVMStructTypeInContext(gallivm->context, elem_types,
                                          ARRAY_SIZE(elem_types), 0);

   (void) target; /* silence unused var warning for non-debug build */
   LP_CHECK_MEMBER_OFFSET(struct draw_tes_jit_context, constants,
                          target, context_type, DRAW_TCS_JIT_CTX_CONSTANTS);
   LP_CHECK_MEMBER_OFFSET(struct draw_tes_jit_context, num_constants,
                          target, context_type, DRAW_TCS_JIT_CTX_NUM_CONSTANTS);
   LP_CHECK_MEMBER_OFFSET(struct draw_tes_jit_context, textures,
                          target, context_type,
                          DRAW_TCS_JIT_CTX_TEXTURES);
   LP_CHECK_MEMBER_OFFSET(struct draw_tes_jit_context, samplers,
                          target, context_type,
                          DRAW_TCS_JIT_CTX_SAMPLERS);
   LP_CHECK_MEMBER_OFFSET(struct draw_tes_jit_context, ssbos,
                          target, context_type, DRAW_TCS_JIT_CTX_SSBOS);
   LP_CHECK_MEMBER_OFFSET(struct draw_tes_jit_context, num_ssbos,
                          target, context_type, DRAW_TCS_JIT_CTX_NUM_SSBOS);
   LP_CHECK_MEMBER_OFFSET(struct draw_tes_jit_context, images,
                          target, context_type, DRAW_TCS_JIT_CTX_IMAGES);
   LP_CHECK_STRUCT_SIZE(struct draw_tes_jit_context,
                        target, context_type);

   return context_type;
}
704 705 706 707 708

/**
 * Create LLVM types for various structures.
 */
static void
709
create_jit_types(struct draw_llvm_variant *variant)
710
{
711
   struct gallivm_state *gallivm = variant->gallivm;
712
   LLVMTypeRef texture_type, sampler_type, context_type, buffer_type,
713
      vb_type, image_type;
714

715
   texture_type = create_jit_texture_type(gallivm, "texture");
716
   sampler_type = create_jit_sampler_type(gallivm, "sampler");
717
   image_type = create_jit_image_type(gallivm, "image");
718

719
   context_type = create_jit_context_type(gallivm, texture_type, sampler_type,
720
                                          image_type,
721
                                          "draw_jit_context");
722
   variant->context_ptr_type = LLVMPointerType(context_type, 0);
723

724
   buffer_type = create_jit_dvbuffer_type(gallivm, "draw_vertex_buffer");
725
   variant->buffer_ptr_type = LLVMPointerType(buffer_type, 0);
726
   
727
   vb_type = create_jit_vertex_buffer_type(gallivm, "pipe_vertex_buffer");
728
   variant->vb_ptr_type = LLVMPointerType(vb_type, 0);
729
}
730 731


732
static LLVMTypeRef
733
get_context_ptr_type(struct draw_llvm_variant *variant)
734
{
735 736 737
   if (!variant->context_ptr_type)
      create_jit_types(variant);
   return variant->context_ptr_type;
738
}
739

Jose Fonseca's avatar
Jose Fonseca committed
740

741
static LLVMTypeRef
742
get_buffer_ptr_type(struct draw_llvm_variant *variant)
743
{
744 745 746
   if (!variant->buffer_ptr_type)
      create_jit_types(variant);
   return variant->buffer_ptr_type;
747
}
748 749


750
static LLVMTypeRef
751
get_vb_ptr_type(struct draw_llvm_variant *variant)
752
{
753 754 755
   if (!variant->vb_ptr_type)
      create_jit_types(variant);
   return variant->vb_ptr_type;
756
}
757

758
static LLVMTypeRef
759
get_vertex_header_ptr_type(struct draw_llvm_variant *variant)
760
{
761 762 763
   if (!variant->vertex_header_ptr_type)
      create_jit_types(variant);
   return variant->vertex_header_ptr_type;
764
}
765

766 767 768 769 770

/**
 * Create per-context LLVM info.
 */
struct draw_llvm *
771
draw_llvm_create(struct draw_context *draw, LLVMContextRef context)
772 773 774
{
   struct draw_llvm *llvm;

775 776 777
   if (!lp_build_init())
      return NULL;

778 779 780 781 782
   llvm = CALLOC_STRUCT( draw_llvm );
   if (!llvm)
      return NULL;

   llvm->draw = draw;
783

784 785 786 787 788
   llvm->context = context;
   if (!llvm->context) {
      llvm->context = LLVMContextCreate();
      llvm->context_owned = true;
   }
789 790 791
   if (!llvm->context)
      goto fail;

792 793 794
   llvm->nr_variants = 0;
   make_empty_list(&llvm->vs_variants_list);

795 796 797
   llvm->nr_gs_variants = 0;
   make_empty_list(&llvm->gs_variants_list);

Dave Airlie's avatar
Dave Airlie committed
798 799 800 801 802 803
   llvm->nr_tcs_variants = 0;
   make_empty_list(&llvm->tcs_variants_list);

   llvm->nr_tes_variants = 0;
   make_empty_list(&llvm->tes_variants_list);

804
   return llvm;
805 806 807 808

fail:
   draw_llvm_destroy(llvm);
   return NULL;
809 810
}

811 812 813 814

/**
 * Free per-context LLVM info.
 */
815 816 817
void
draw_llvm_destroy(struct draw_llvm *llvm)
{
818 819
   if (llvm->context_owned)
      LLVMContextDispose(llvm->context);
820 821
   llvm->context = NULL;

822
   /* XXX free other draw_llvm data? */
823
   FREE(llvm);
824 825
}

826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849
static void
draw_get_ir_cache_key(struct nir_shader *nir,
                      const void *key, size_t key_size,
                      uint32_t val_32bit,
                      unsigned char ir_sha1_cache_key[20])
{
   struct blob blob = { 0 };
   unsigned ir_size;
   void *ir_binary;

   blob_init(&blob);
   nir_serialize(&blob, nir, true);
   ir_binary = blob.data;
   ir_size = blob.size;

   struct mesa_sha1 ctx;
   _mesa_sha1_init(&ctx);
   _mesa_sha1_update(&ctx, key, key_size);
   _mesa_sha1_update(&ctx, ir_binary, ir_size);
   _mesa_sha1_update(&ctx, &val_32bit, 4);
   _mesa_sha1_final(&ctx, ir_sha1_cache_key);

   blob_finish(&blob);
}
850 851 852 853

/**
 * Create LLVM-generated code for a vertex shader.
 */
854
struct draw_llvm_variant *
855
draw_llvm_create_variant(struct draw_llvm *llvm,
856 857
                         unsigned num_inputs,
                         const struct draw_llvm_variant_key *key)
858
{
859
   struct draw_llvm_variant *variant;
860 861
   struct llvm_vertex_shader *shader =
      llvm_vertex_shader(llvm->draw->vs.vertex_shader);
862
   LLVMTypeRef vertex_header;
863
   char module_name[64];
864 865 866
   unsigned char ir_sha1_cache_key[20];
   struct lp_cached_code cached = { 0 };
   bool needs_caching = false;
867
   variant = MALLOC(sizeof *variant +
868 869
                    shader->variant_key_size -
                    sizeof variant->key);
870
   if (!variant)
871 872
      return NULL;

873
   variant->llvm = llvm;
874
   variant->shader = shader;
875
   memcpy(&variant->key, key, shader->variant_key_size);
876

877 878
   snprintf(module_name, sizeof(module_name), "draw_llvm_vs_variant%u",
            variant->shader->variants_cached);
879

880 881 882 883 884 885
   if (shader->base.state.ir.nir && llvm->draw->disk_cache_cookie) {
      draw_get_ir_cache_key(shader->base.state.ir.nir,
                            key,
                            shader->variant_key_size,
                            num_inputs,
                            ir_sha1_cache_key);
886

887 888 889 890 891 892 893
      llvm->draw->disk_cache_find_shader(llvm->draw->disk_cache_cookie,
                                         &cached,
                                         ir_sha1_cache_key);
      if (!cached.data_size)
         needs_caching = true;
   }
   variant->gallivm = gallivm_create(module_name, llvm->context, &cached);
894

895
   create_jit_types(variant);
896

897
   if (gallivm_debug & (GALLIVM_DEBUG_TGSI | GALLIVM_DEBUG_IR)) {
898 899 900 901
      if (llvm->draw->vs.vertex_shader->state.type == PIPE_SHADER_IR_TGSI)
         tgsi_dump(llvm->draw->vs.vertex_shader->state.tokens, 0);
      else
         nir_print_shader(llvm->draw->vs.vertex_shader->state.ir.nir, stderr);
902 903 904
      draw_llvm_dump_variant_key(&variant->key);
   }

905
   vertex_header = create_jit_vertex_header(variant->gallivm, num_inputs);
906

907
   variant->vertex_header_ptr_type = LLVMPointerType(vertex_header, 0);
908

909
   draw_llvm_generate(llvm, variant);
910

911 912 913 914 915
   gallivm_compile_module(variant->gallivm);

   variant->jit_func = (draw_jit_vert_func)
         gallivm_jit_function(variant->gallivm, variant->function);

916 917 918 919
   if (needs_caching)
      llvm->draw->disk_cache_insert_shader(llvm->draw->disk_cache_cookie,
                                           &cached,
                                           ir_sha1_cache_key);
920 921
   gallivm_free_ir(variant->gallivm);

922 923 924 925 926
   variant->list_item_global.base = variant;
   variant->list_item_local.base = variant;
   /*variant->no = */shader->variants_created++;
   variant->list_item_global.base = variant;

927
   return variant;
928 929
}

930

931
static void
932
generate_vs(struct draw_llvm_variant *variant,
933
            LLVMBuilderRef builder,
934
            struct lp_type vs_type,
935 936
            LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS],
            const LLVMValueRef (*inputs)[TGSI_NUM_CHANNELS],
937
            const struct lp_bld_tgsi_system_values *system_values,
938
            LLVMValueRef context_ptr,
939
            const struct lp_build_sampler_soa *draw_sampler,
Dave Airlie's avatar
Dave Airlie committed
940
            const struct lp_build_image_soa *draw_image,
941 942
            boolean clamp_vertex_color,
            struct lp_build_mask_context *bld_mask)
943
{
944
   struct draw_llvm *llvm = variant->llvm;
945
   const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens;
946 947 948 949
   LLVMValueRef consts_ptr =
      draw_jit_context_vs_constants(variant->gallivm, context_ptr);
   LLVMValueRef num_consts_ptr =
      draw_jit_context_num_vs_constants(variant->gallivm, context_ptr);
950 951 952 953
   LLVMValueRef ssbos_ptr =
      draw_jit_context_vs_ssbos(variant->gallivm, context_ptr);
   LLVMValueRef num_ssbos_ptr =
      draw_jit_context_num_vs_ssbos(variant->gallivm, context_ptr);
954

955 956
   struct lp_build_tgsi_params params;
   memset(&params, 0, sizeof(params));
957 958 959 960 961 962 963 964 965 966 967 968

   params.type = vs_type;
   params.mask = bld_mask;
   params.consts_ptr = consts_ptr;
   params.const_sizes_ptr = num_consts_ptr;
   params.system_values = system_values;
   params.inputs = inputs;
   params.context_ptr = context_ptr;
   params.sampler = draw_sampler;
   params.info = &llvm->draw->vs.vertex_shader->info;
   params.ssbo_ptr = ssbos_ptr;
   params.ssbo_sizes_ptr = num_ssbos_ptr;
Dave Airlie's avatar
Dave Airlie committed
969
   params.image = draw_image;
970

971 972 973 974 975 976 977 978 979 980 981
   if (llvm->draw->vs.vertex_shader->state.ir.nir &&
       llvm->draw->vs.vertex_shader->state.type == PIPE_SHADER_IR_NIR)
      lp_build_nir_soa(variant->gallivm,
                       llvm->draw->vs.vertex_shader->state.ir.nir,
                       &params,
                       outputs);
   else
      lp_build_tgsi_soa(variant->gallivm,
                        tokens,
                        &params,
                        outputs);
982

983
   {
984 985 986 987
      LLVMValueRef out;
      unsigned chan, attrib;
      struct lp_build_context bld;
      struct tgsi_shader_info* info = &llvm->draw->vs.vertex_shader->info;
988
      lp_build_context_init(&bld, variant->gallivm, vs_type);
989 990

      for (attrib = 0; attrib < info->num_outputs; ++attrib) {
991
         for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
992
            if (outputs[attrib][chan]) {
993 994 995
               switch (info->output_semantic_name[attrib]) {
               case TGSI_SEMANTIC_COLOR:
               case TGSI_SEMANTIC_BCOLOR:
996 997 998 999 1000 1001
                  if (clamp_vertex_color) {
                     out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
                     out = lp_build_clamp(&bld, out, bld.zero, bld.one);
                     LLVMBuildStore(builder, out, outputs[attrib][chan]);
                  }
                  break;
1002 1003 1004 1005 1006
               }
            }
         }
      }
   }
1007 1008
}

1009

1010
static void
1011 1012 1013 1014 1015 1016 1017 1018
fetch_instanced(struct gallivm_state *gallivm,
                const struct util_format_description *format_desc,
                struct lp_type vs_type,
                LLVMValueRef vb_stride,
                LLVMValueRef map_ptr,
                LLVMValueRef buffer_size_adj,
                LLVMValueRef *inputs,
                LLVMValueRef index)
1019
{
1020 1021 1022
   LLVMTypeRef i32_t = LLVMInt32TypeInContext(gallivm->context);
   LLVMTypeRef aosf_t, aosi_t;
   LLVMValueRef zero = LLVMConstNull(i32_t);
1023
   LLVMBuilderRef builder = gallivm->builder;
1024
   LLVMValueRef stride, buffer_overflowed, aos, index_valid;
1025
   unsigned i;
1026

1027 1028 1029
   aosf_t = lp_build_vec_type(gallivm, lp_float32_vec4_type());
   aosi_t = lp_build_vec_type(gallivm, lp_int32_vec4_type());

1030 1031
   /* This mul can overflow. Wraparound is ok. */
   stride = LLVMBuildMul(builder, vb_stride, index, "");
1032

1033
   buffer_overflowed = LLVMBuildICmp(builder, LLVMIntUGE,
1034
                                     stride, buffer_size_adj,
1035
                                     "buffer_overflowed");
1036 1037

   if (0) {
1038
      lp_build_print_value(gallivm, "   instance index = ", index);
1039 1040
      lp_build_print_value(gallivm, "   buffer overflowed = ", buffer_overflowed);
   }
1041

1042 1043 1044
   index_valid = LLVMBuildNot(builder, buffer_overflowed, "");
   index_valid = LLVMBuildSExt(builder, index_valid, i32_t, "");
   stride = LLVMBuildAnd(builder, stride, index_valid, "");
1045

1046 1047 1048 1049 1050 1051 1052 1053 1054