ir3.c 27.4 KB
Newer Older
Rob Clark's avatar
Rob Clark committed
1
/*
2
 * Copyright (c) 2012 Rob Clark <robdclark@gmail.com>
Rob Clark's avatar
Rob Clark committed
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

24
#include "ir3.h"
Rob Clark's avatar
Rob Clark committed
25 26 27 28 29 30 31 32

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include <stdbool.h>
#include <errno.h>

33
#include "util/bitscan.h"
34
#include "util/ralloc.h"
35
#include "util/u_math.h"
36

Rob Clark's avatar
Rob Clark committed
37 38 39 40 41
#include "instr-a3xx.h"

/* simple allocator to carve allocations out of an up-front allocated heap,
 * so that we can free everything easily in one shot.
 */
42
void * ir3_alloc(struct ir3 *shader, int sz)
Rob Clark's avatar
Rob Clark committed
43
{
44
	return rzalloc_size(shader, sz); /* TODO: don't use rzalloc */
Rob Clark's avatar
Rob Clark committed
45 46
}

47 48
struct ir3 * ir3_create(struct ir3_compiler *compiler,
		unsigned nin, unsigned nout)
Rob Clark's avatar
Rob Clark committed
49
{
50
	struct ir3 *shader = rzalloc(compiler, struct ir3);
51

52
	shader->compiler = compiler;
53 54 55 56 57 58
	shader->ninputs = nin;
	shader->inputs = ir3_alloc(shader, sizeof(shader->inputs[0]) * nin);

	shader->noutputs = nout;
	shader->outputs = ir3_alloc(shader, sizeof(shader->outputs[0]) * nout);

59
	list_inithead(&shader->block_list);
Rob Clark's avatar
Rob Clark committed
60
	list_inithead(&shader->array_list);
61

62
	return shader;
Rob Clark's avatar
Rob Clark committed
63 64
}

65
void ir3_destroy(struct ir3 *shader)
Rob Clark's avatar
Rob Clark committed
66
{
67
	ralloc_free(shader);
Rob Clark's avatar
Rob Clark committed
68 69 70 71
}

#define iassert(cond) do { \
	if (!(cond)) { \
72
		debug_assert(cond); \
Rob Clark's avatar
Rob Clark committed
73 74 75
		return -1; \
	} } while (0)

76 77 78 79 80 81 82
#define iassert_type(reg, full) do { \
	if ((full)) { \
		iassert(!((reg)->flags & IR3_REG_HALF)); \
	} else { \
		iassert((reg)->flags & IR3_REG_HALF); \
	} } while (0);

83
static uint32_t reg(struct ir3_register *reg, struct ir3_info *info,
Rob Clark's avatar
Rob Clark committed
84 85 86 87
		uint32_t repeat, uint32_t valid_flags)
{
	reg_t val = { .dummy32 = 0 };

88 89 90 91
	if (reg->flags & ~valid_flags) {
		debug_printf("INVALID FLAGS: %x vs %x\n",
				reg->flags, valid_flags);
	}
Rob Clark's avatar
Rob Clark committed
92 93 94 95 96 97 98

	if (!(reg->flags & IR3_REG_R))
		repeat = 0;

	if (reg->flags & IR3_REG_IMMED) {
		val.iim_val = reg->iim_val;
	} else {
99
		unsigned components;
Rob Clark's avatar
Rob Clark committed
100
		int16_t max;
101 102 103

		if (reg->flags & IR3_REG_RELATIV) {
			components = reg->size;
104
			val.idummy10 = reg->array.offset;
Rob Clark's avatar
Rob Clark committed
105
			max = (reg->array.offset + repeat + components - 1) >> 2;
106 107 108 109
		} else {
			components = util_last_bit(reg->wrmask);
			val.comp = reg->num & 0x3;
			val.num  = reg->num >> 2;
Rob Clark's avatar
Rob Clark committed
110
			max = (reg->num + repeat + components - 1) >> 2;
111
		}
Rob Clark's avatar
Rob Clark committed
112 113 114

		if (reg->flags & IR3_REG_CONST) {
			info->max_const = MAX2(info->max_const, max);
115 116
		} else if (val.num == 63) {
			/* ignore writes to dummy register r63.x */
117
		} else if (max < 48) {
Rob Clark's avatar
Rob Clark committed
118
			if (reg->flags & IR3_REG_HALF) {
119 120 121 122 123 124
				if (info->gpu_id >= 600) {
					/* starting w/ a6xx, half regs conflict with full regs: */
					info->max_reg = MAX2(info->max_reg, (max+1)/2);
				} else {
					info->max_half_reg = MAX2(info->max_half_reg, max);
				}
Rob Clark's avatar
Rob Clark committed
125 126 127 128 129 130 131 132 133 134
			} else {
				info->max_reg = MAX2(info->max_reg, max);
			}
		}
	}

	return val.dummy32;
}

static int emit_cat0(struct ir3_instruction *instr, void *ptr,
135
		struct ir3_info *info)
Rob Clark's avatar
Rob Clark committed
136 137 138
{
	instr_cat0_t *cat0 = ptr;

139 140 141
	if (info->gpu_id >= 500) {
		cat0->a5xx.immed = instr->cat0.immed;
	} else if (info->gpu_id >= 400) {
142 143 144 145
		cat0->a4xx.immed = instr->cat0.immed;
	} else {
		cat0->a3xx.immed = instr->cat0.immed;
	}
Rob Clark's avatar
Rob Clark committed
146 147 148 149 150 151 152 153 154 155 156 157 158
	cat0->repeat   = instr->repeat;
	cat0->ss       = !!(instr->flags & IR3_INSTR_SS);
	cat0->inv      = instr->cat0.inv;
	cat0->comp     = instr->cat0.comp;
	cat0->opc      = instr->opc;
	cat0->jmp_tgt  = !!(instr->flags & IR3_INSTR_JP);
	cat0->sync     = !!(instr->flags & IR3_INSTR_SY);
	cat0->opc_cat  = 0;

	return 0;
}

static int emit_cat1(struct ir3_instruction *instr, void *ptr,
159
		struct ir3_info *info)
Rob Clark's avatar
Rob Clark committed
160 161 162 163 164 165
{
	struct ir3_register *dst = instr->regs[0];
	struct ir3_register *src = instr->regs[1];
	instr_cat1_t *cat1 = ptr;

	iassert(instr->regs_count == 2);
166 167 168
	iassert_type(dst, type_size(instr->cat1.dst_type) == 32);
	if (!(src->flags & IR3_REG_IMMED))
		iassert_type(src, type_size(instr->cat1.src_type) == 32);
Rob Clark's avatar
Rob Clark committed
169 170 171 172 173

	if (src->flags & IR3_REG_IMMED) {
		cat1->iim_val = src->iim_val;
		cat1->src_im  = 1;
	} else if (src->flags & IR3_REG_RELATIV) {
174 175
		cat1->off       = reg(src, info, instr->repeat,
				IR3_REG_R | IR3_REG_CONST | IR3_REG_HALF | IR3_REG_RELATIV);
Rob Clark's avatar
Rob Clark committed
176
		cat1->src_rel   = 1;
177
		cat1->src_rel_c = !!(src->flags & IR3_REG_CONST);
Rob Clark's avatar
Rob Clark committed
178 179
	} else {
		cat1->src  = reg(src, info, instr->repeat,
180
				IR3_REG_R | IR3_REG_CONST | IR3_REG_HALF);
181
		cat1->src_c     = !!(src->flags & IR3_REG_CONST);
Rob Clark's avatar
Rob Clark committed
182 183 184 185 186 187 188 189
	}

	cat1->dst      = reg(dst, info, instr->repeat,
			IR3_REG_RELATIV | IR3_REG_EVEN |
			IR3_REG_R | IR3_REG_POS_INF | IR3_REG_HALF);
	cat1->repeat   = instr->repeat;
	cat1->src_r    = !!(src->flags & IR3_REG_R);
	cat1->ss       = !!(instr->flags & IR3_INSTR_SS);
190
	cat1->ul       = !!(instr->flags & IR3_INSTR_UL);
Rob Clark's avatar
Rob Clark committed
191 192 193 194 195 196 197 198 199 200 201 202 203
	cat1->dst_type = instr->cat1.dst_type;
	cat1->dst_rel  = !!(dst->flags & IR3_REG_RELATIV);
	cat1->src_type = instr->cat1.src_type;
	cat1->even     = !!(dst->flags & IR3_REG_EVEN);
	cat1->pos_inf  = !!(dst->flags & IR3_REG_POS_INF);
	cat1->jmp_tgt  = !!(instr->flags & IR3_INSTR_JP);
	cat1->sync     = !!(instr->flags & IR3_INSTR_SY);
	cat1->opc_cat  = 1;

	return 0;
}

static int emit_cat2(struct ir3_instruction *instr, void *ptr,
204
		struct ir3_info *info)
Rob Clark's avatar
Rob Clark committed
205 206 207 208 209
{
	struct ir3_register *dst = instr->regs[0];
	struct ir3_register *src1 = instr->regs[1];
	struct ir3_register *src2 = instr->regs[2];
	instr_cat2_t *cat2 = ptr;
210
	unsigned absneg = ir3_cat2_absneg(instr->opc);
Rob Clark's avatar
Rob Clark committed
211 212 213

	iassert((instr->regs_count == 2) || (instr->regs_count == 3));

214
	if (src1->flags & IR3_REG_RELATIV) {
Rob Clark's avatar
Rob Clark committed
215
		iassert(src1->array.offset < (1 << 10));
216
		cat2->rel1.src1      = reg(src1, info, instr->repeat,
217 218
				IR3_REG_RELATIV | IR3_REG_CONST | IR3_REG_R |
				IR3_REG_HALF | absneg);
219 220 221 222 223
		cat2->rel1.src1_c    = !!(src1->flags & IR3_REG_CONST);
		cat2->rel1.src1_rel  = 1;
	} else if (src1->flags & IR3_REG_CONST) {
		iassert(src1->num < (1 << 12));
		cat2->c1.src1   = reg(src1, info, instr->repeat,
224
				IR3_REG_CONST | IR3_REG_R | IR3_REG_HALF);
225 226 227 228
		cat2->c1.src1_c = 1;
	} else {
		iassert(src1->num < (1 << 11));
		cat2->src1 = reg(src1, info, instr->repeat,
229 230
				IR3_REG_IMMED | IR3_REG_R | IR3_REG_HALF |
				absneg);
231
	}
Rob Clark's avatar
Rob Clark committed
232
	cat2->src1_im  = !!(src1->flags & IR3_REG_IMMED);
233 234
	cat2->src1_neg = !!(src1->flags & (IR3_REG_FNEG | IR3_REG_SNEG | IR3_REG_BNOT));
	cat2->src1_abs = !!(src1->flags & (IR3_REG_FABS | IR3_REG_SABS));
Rob Clark's avatar
Rob Clark committed
235 236 237 238 239
	cat2->src1_r   = !!(src1->flags & IR3_REG_R);

	if (src2) {
		iassert((src2->flags & IR3_REG_IMMED) ||
				!((src1->flags ^ src2->flags) & IR3_REG_HALF));
240 241

		if (src2->flags & IR3_REG_RELATIV) {
Rob Clark's avatar
Rob Clark committed
242
			iassert(src2->array.offset < (1 << 10));
243
			cat2->rel2.src2      = reg(src2, info, instr->repeat,
244 245
					IR3_REG_RELATIV | IR3_REG_CONST | IR3_REG_R |
					IR3_REG_HALF | absneg);
246 247 248 249 250
			cat2->rel2.src2_c    = !!(src2->flags & IR3_REG_CONST);
			cat2->rel2.src2_rel  = 1;
		} else if (src2->flags & IR3_REG_CONST) {
			iassert(src2->num < (1 << 12));
			cat2->c2.src2   = reg(src2, info, instr->repeat,
251
					IR3_REG_CONST | IR3_REG_R | IR3_REG_HALF);
252 253 254 255
			cat2->c2.src2_c = 1;
		} else {
			iassert(src2->num < (1 << 11));
			cat2->src2 = reg(src2, info, instr->repeat,
256 257
					IR3_REG_IMMED | IR3_REG_R | IR3_REG_HALF |
					absneg);
258 259
		}

Rob Clark's avatar
Rob Clark committed
260
		cat2->src2_im  = !!(src2->flags & IR3_REG_IMMED);
261 262
		cat2->src2_neg = !!(src2->flags & (IR3_REG_FNEG | IR3_REG_SNEG | IR3_REG_BNOT));
		cat2->src2_abs = !!(src2->flags & (IR3_REG_FABS | IR3_REG_SABS));
Rob Clark's avatar
Rob Clark committed
263 264 265 266 267 268
		cat2->src2_r   = !!(src2->flags & IR3_REG_R);
	}

	cat2->dst      = reg(dst, info, instr->repeat,
			IR3_REG_R | IR3_REG_EI | IR3_REG_HALF);
	cat2->repeat   = instr->repeat;
269
	cat2->sat      = !!(instr->flags & IR3_INSTR_SAT);
Rob Clark's avatar
Rob Clark committed
270 271 272 273 274 275 276 277 278 279 280 281 282 283 284
	cat2->ss       = !!(instr->flags & IR3_INSTR_SS);
	cat2->ul       = !!(instr->flags & IR3_INSTR_UL);
	cat2->dst_half = !!((src1->flags ^ dst->flags) & IR3_REG_HALF);
	cat2->ei       = !!(dst->flags & IR3_REG_EI);
	cat2->cond     = instr->cat2.condition;
	cat2->full     = ! (src1->flags & IR3_REG_HALF);
	cat2->opc      = instr->opc;
	cat2->jmp_tgt  = !!(instr->flags & IR3_INSTR_JP);
	cat2->sync     = !!(instr->flags & IR3_INSTR_SY);
	cat2->opc_cat  = 2;

	return 0;
}

static int emit_cat3(struct ir3_instruction *instr, void *ptr,
285
		struct ir3_info *info)
Rob Clark's avatar
Rob Clark committed
286 287 288 289 290
{
	struct ir3_register *dst = instr->regs[0];
	struct ir3_register *src1 = instr->regs[1];
	struct ir3_register *src2 = instr->regs[2];
	struct ir3_register *src3 = instr->regs[3];
291
	unsigned absneg = ir3_cat3_absneg(instr->opc);
Rob Clark's avatar
Rob Clark committed
292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314
	instr_cat3_t *cat3 = ptr;
	uint32_t src_flags = 0;

	switch (instr->opc) {
	case OPC_MAD_F16:
	case OPC_MAD_U16:
	case OPC_MAD_S16:
	case OPC_SEL_B16:
	case OPC_SEL_S16:
	case OPC_SEL_F16:
	case OPC_SAD_S16:
	case OPC_SAD_S32:  // really??
		src_flags |= IR3_REG_HALF;
		break;
	default:
		break;
	}

	iassert(instr->regs_count == 4);
	iassert(!((src1->flags ^ src_flags) & IR3_REG_HALF));
	iassert(!((src2->flags ^ src_flags) & IR3_REG_HALF));
	iassert(!((src3->flags ^ src_flags) & IR3_REG_HALF));

315
	if (src1->flags & IR3_REG_RELATIV) {
Rob Clark's avatar
Rob Clark committed
316
		iassert(src1->array.offset < (1 << 10));
317
		cat3->rel1.src1      = reg(src1, info, instr->repeat,
318 319
				IR3_REG_RELATIV | IR3_REG_CONST | IR3_REG_R |
				IR3_REG_HALF | absneg);
320 321 322 323 324
		cat3->rel1.src1_c    = !!(src1->flags & IR3_REG_CONST);
		cat3->rel1.src1_rel  = 1;
	} else if (src1->flags & IR3_REG_CONST) {
		iassert(src1->num < (1 << 12));
		cat3->c1.src1   = reg(src1, info, instr->repeat,
325
				IR3_REG_CONST | IR3_REG_R | IR3_REG_HALF);
326 327 328 329
		cat3->c1.src1_c = 1;
	} else {
		iassert(src1->num < (1 << 11));
		cat3->src1 = reg(src1, info, instr->repeat,
330
				IR3_REG_R | IR3_REG_HALF | absneg);
331 332
	}

333
	cat3->src1_neg = !!(src1->flags & (IR3_REG_FNEG | IR3_REG_SNEG | IR3_REG_BNOT));
Rob Clark's avatar
Rob Clark committed
334 335 336
	cat3->src1_r   = !!(src1->flags & IR3_REG_R);

	cat3->src2     = reg(src2, info, instr->repeat,
337
			IR3_REG_CONST | IR3_REG_R | IR3_REG_HALF | absneg);
Rob Clark's avatar
Rob Clark committed
338
	cat3->src2_c   = !!(src2->flags & IR3_REG_CONST);
339
	cat3->src2_neg = !!(src2->flags & (IR3_REG_FNEG | IR3_REG_SNEG | IR3_REG_BNOT));
Rob Clark's avatar
Rob Clark committed
340 341
	cat3->src2_r   = !!(src2->flags & IR3_REG_R);

342 343

	if (src3->flags & IR3_REG_RELATIV) {
Rob Clark's avatar
Rob Clark committed
344
		iassert(src3->array.offset < (1 << 10));
345
		cat3->rel2.src3      = reg(src3, info, instr->repeat,
346 347
				IR3_REG_RELATIV | IR3_REG_CONST | IR3_REG_R |
				IR3_REG_HALF | absneg);
348 349 350 351 352
		cat3->rel2.src3_c    = !!(src3->flags & IR3_REG_CONST);
		cat3->rel2.src3_rel  = 1;
	} else if (src3->flags & IR3_REG_CONST) {
		iassert(src3->num < (1 << 12));
		cat3->c2.src3   = reg(src3, info, instr->repeat,
353
				IR3_REG_CONST | IR3_REG_R | IR3_REG_HALF);
354 355 356 357
		cat3->c2.src3_c = 1;
	} else {
		iassert(src3->num < (1 << 11));
		cat3->src3 = reg(src3, info, instr->repeat,
358
				IR3_REG_R | IR3_REG_HALF | absneg);
359 360
	}

361
	cat3->src3_neg = !!(src3->flags & (IR3_REG_FNEG | IR3_REG_SNEG | IR3_REG_BNOT));
Rob Clark's avatar
Rob Clark committed
362 363 364 365
	cat3->src3_r   = !!(src3->flags & IR3_REG_R);

	cat3->dst      = reg(dst, info, instr->repeat, IR3_REG_R | IR3_REG_HALF);
	cat3->repeat   = instr->repeat;
366
	cat3->sat      = !!(instr->flags & IR3_INSTR_SAT);
Rob Clark's avatar
Rob Clark committed
367 368 369 370 371 372 373 374 375 376 377 378
	cat3->ss       = !!(instr->flags & IR3_INSTR_SS);
	cat3->ul       = !!(instr->flags & IR3_INSTR_UL);
	cat3->dst_half = !!((src_flags ^ dst->flags) & IR3_REG_HALF);
	cat3->opc      = instr->opc;
	cat3->jmp_tgt  = !!(instr->flags & IR3_INSTR_JP);
	cat3->sync     = !!(instr->flags & IR3_INSTR_SY);
	cat3->opc_cat  = 3;

	return 0;
}

static int emit_cat4(struct ir3_instruction *instr, void *ptr,
379
		struct ir3_info *info)
Rob Clark's avatar
Rob Clark committed
380 381 382 383 384 385 386
{
	struct ir3_register *dst = instr->regs[0];
	struct ir3_register *src = instr->regs[1];
	instr_cat4_t *cat4 = ptr;

	iassert(instr->regs_count == 2);

387
	if (src->flags & IR3_REG_RELATIV) {
Rob Clark's avatar
Rob Clark committed
388
		iassert(src->array.offset < (1 << 10));
389
		cat4->rel.src      = reg(src, info, instr->repeat,
390 391
				IR3_REG_RELATIV | IR3_REG_CONST | IR3_REG_FNEG |
				IR3_REG_FABS | IR3_REG_R | IR3_REG_HALF);
392 393 394 395 396
		cat4->rel.src_c    = !!(src->flags & IR3_REG_CONST);
		cat4->rel.src_rel  = 1;
	} else if (src->flags & IR3_REG_CONST) {
		iassert(src->num < (1 << 12));
		cat4->c.src   = reg(src, info, instr->repeat,
397
				IR3_REG_CONST | IR3_REG_FNEG | IR3_REG_FABS |
398 399 400 401 402
				IR3_REG_R | IR3_REG_HALF);
		cat4->c.src_c = 1;
	} else {
		iassert(src->num < (1 << 11));
		cat4->src = reg(src, info, instr->repeat,
403
				IR3_REG_IMMED | IR3_REG_FNEG | IR3_REG_FABS |
404 405 406
				IR3_REG_R | IR3_REG_HALF);
	}

Rob Clark's avatar
Rob Clark committed
407
	cat4->src_im   = !!(src->flags & IR3_REG_IMMED);
408 409
	cat4->src_neg  = !!(src->flags & IR3_REG_FNEG);
	cat4->src_abs  = !!(src->flags & IR3_REG_FABS);
Rob Clark's avatar
Rob Clark committed
410 411 412 413
	cat4->src_r    = !!(src->flags & IR3_REG_R);

	cat4->dst      = reg(dst, info, instr->repeat, IR3_REG_R | IR3_REG_HALF);
	cat4->repeat   = instr->repeat;
414
	cat4->sat      = !!(instr->flags & IR3_INSTR_SAT);
Rob Clark's avatar
Rob Clark committed
415 416 417 418 419 420 421 422 423 424 425 426 427
	cat4->ss       = !!(instr->flags & IR3_INSTR_SS);
	cat4->ul       = !!(instr->flags & IR3_INSTR_UL);
	cat4->dst_half = !!((src->flags ^ dst->flags) & IR3_REG_HALF);
	cat4->full     = ! (src->flags & IR3_REG_HALF);
	cat4->opc      = instr->opc;
	cat4->jmp_tgt  = !!(instr->flags & IR3_INSTR_JP);
	cat4->sync     = !!(instr->flags & IR3_INSTR_SY);
	cat4->opc_cat  = 4;

	return 0;
}

static int emit_cat5(struct ir3_instruction *instr, void *ptr,
428
		struct ir3_info *info)
Rob Clark's avatar
Rob Clark committed
429 430 431 432 433 434 435
{
	struct ir3_register *dst = instr->regs[0];
	struct ir3_register *src1 = instr->regs[1];
	struct ir3_register *src2 = instr->regs[2];
	struct ir3_register *src3 = instr->regs[3];
	instr_cat5_t *cat5 = ptr;

436
	iassert_type(dst, type_size(instr->cat5.type) == 32)
Rob Clark's avatar
Rob Clark committed
437

438 439 440
	assume(src1 || !src2);
	assume(src2 || !src3);

Rob Clark's avatar
Rob Clark committed
441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483
	if (src1) {
		cat5->full = ! (src1->flags & IR3_REG_HALF);
		cat5->src1 = reg(src1, info, instr->repeat, IR3_REG_HALF);
	}

	if (instr->flags & IR3_INSTR_S2EN) {
		if (src2) {
			iassert(!((src1->flags ^ src2->flags) & IR3_REG_HALF));
			cat5->s2en.src2 = reg(src2, info, instr->repeat, IR3_REG_HALF);
		}
		if (src3) {
			iassert(src3->flags & IR3_REG_HALF);
			cat5->s2en.src3 = reg(src3, info, instr->repeat, IR3_REG_HALF);
		}
		iassert(!(instr->cat5.samp | instr->cat5.tex));
	} else {
		iassert(!src3);
		if (src2) {
			iassert(!((src1->flags ^ src2->flags) & IR3_REG_HALF));
			cat5->norm.src2 = reg(src2, info, instr->repeat, IR3_REG_HALF);
		}
		cat5->norm.samp = instr->cat5.samp;
		cat5->norm.tex  = instr->cat5.tex;
	}

	cat5->dst      = reg(dst, info, instr->repeat, IR3_REG_R | IR3_REG_HALF);
	cat5->wrmask   = dst->wrmask;
	cat5->type     = instr->cat5.type;
	cat5->is_3d    = !!(instr->flags & IR3_INSTR_3D);
	cat5->is_a     = !!(instr->flags & IR3_INSTR_A);
	cat5->is_s     = !!(instr->flags & IR3_INSTR_S);
	cat5->is_s2en  = !!(instr->flags & IR3_INSTR_S2EN);
	cat5->is_o     = !!(instr->flags & IR3_INSTR_O);
	cat5->is_p     = !!(instr->flags & IR3_INSTR_P);
	cat5->opc      = instr->opc;
	cat5->jmp_tgt  = !!(instr->flags & IR3_INSTR_JP);
	cat5->sync     = !!(instr->flags & IR3_INSTR_SY);
	cat5->opc_cat  = 5;

	return 0;
}

static int emit_cat6(struct ir3_instruction *instr, void *ptr,
484
		struct ir3_info *info)
Rob Clark's avatar
Rob Clark committed
485
{
486
	struct ir3_register *dst, *src1, *src2;
Rob Clark's avatar
Rob Clark committed
487
	instr_cat6_t *cat6 = ptr;
488
	bool type_full = type_size(instr->cat6.type) == 32;
Rob Clark's avatar
Rob Clark committed
489

490 491 492 493 494 495 496
	cat6->type     = instr->cat6.type;
	cat6->opc      = instr->opc;
	cat6->jmp_tgt  = !!(instr->flags & IR3_INSTR_JP);
	cat6->sync     = !!(instr->flags & IR3_INSTR_SY);
	cat6->g        = !!(instr->flags & IR3_INSTR_G);
	cat6->opc_cat  = 6;

497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526
	switch (instr->opc) {
	case OPC_RESINFO:
	case OPC_RESFMT:
		iassert_type(instr->regs[0], type_full); /* dst */
		iassert_type(instr->regs[1], type_full); /* src1 */
		break;
	case OPC_L2G:
	case OPC_G2L:
		iassert_type(instr->regs[0], true);      /* dst */
		iassert_type(instr->regs[1], true);      /* src1 */
		break;
	case OPC_STG:
	case OPC_STL:
	case OPC_STP:
	case OPC_STI:
	case OPC_STLW:
	case OPC_STIB:
		/* no dst, so regs[0] is dummy */
		iassert_type(instr->regs[1], true);      /* dst */
		iassert_type(instr->regs[2], type_full); /* src1 */
		iassert_type(instr->regs[3], true);      /* src2 */
		break;
	default:
		iassert_type(instr->regs[0], type_full); /* dst */
		iassert_type(instr->regs[1], true);      /* src1 */
		if (instr->regs_count > 2)
			iassert_type(instr->regs[2], true);  /* src1 */
		break;
	}

527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545
	/* the "dst" for a store instruction is (from the perspective
	 * of data flow in the shader, ie. register use/def, etc) in
	 * fact a register that is read by the instruction, rather
	 * than written:
	 */
	if (is_store(instr)) {
		iassert(instr->regs_count >= 3);

		dst  = instr->regs[1];
		src1 = instr->regs[2];
		src2 = (instr->regs_count >= 4) ? instr->regs[3] : NULL;
	} else {
		iassert(instr->regs_count >= 2);

		dst  = instr->regs[0];
		src1 = instr->regs[1];
		src2 = (instr->regs_count >= 3) ? instr->regs[2] : NULL;
	}

546 547 548 549 550
	/* TODO we need a more comprehensive list about which instructions
	 * can be encoded which way.  Or possibly use IR3_INSTR_0 flag to
	 * indicate to use the src_off encoding even if offset is zero
	 * (but then what to do about dst_off?)
	 */
551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589
	if (is_atomic(instr->opc)) {
		instr_cat6ldgb_t *ldgb = ptr;

		/* maybe these two bits both determine the instruction encoding? */
		cat6->src_off = false;

		ldgb->d = instr->cat6.d - 1;
		ldgb->typed = instr->cat6.typed;
		ldgb->type_size = instr->cat6.iim_val - 1;

		ldgb->dst = reg(dst, info, instr->repeat, IR3_REG_R | IR3_REG_HALF);

		if (ldgb->g) {
			struct ir3_register *src3 = instr->regs[3];
			struct ir3_register *src4 = instr->regs[4];

			/* first src is src_ssbo: */
			iassert(src1->flags & IR3_REG_IMMED);
			ldgb->src_ssbo = src1->uim_val;

			ldgb->src1 = reg(src2, info, instr->repeat, IR3_REG_IMMED);
			ldgb->src1_im = !!(src2->flags & IR3_REG_IMMED);
			ldgb->src2 = reg(src3, info, instr->repeat, IR3_REG_IMMED);
			ldgb->src2_im = !!(src3->flags & IR3_REG_IMMED);

			ldgb->src3 = reg(src4, info, instr->repeat, 0);
			ldgb->pad0 = 0x1;
			ldgb->pad3 = 0x1;
		} else {
			ldgb->src1 = reg(src1, info, instr->repeat, IR3_REG_IMMED);
			ldgb->src1_im = !!(src1->flags & IR3_REG_IMMED);
			ldgb->src2 = reg(src2, info, instr->repeat, IR3_REG_IMMED);
			ldgb->src2_im = !!(src2->flags & IR3_REG_IMMED);
			ldgb->pad0 = 0x1;
			ldgb->pad3 = 0x0;
		}

		return 0;
	} else if (instr->opc == OPC_LDGB) {
590 591 592 593 594 595
		struct ir3_register *src3 = instr->regs[3];
		instr_cat6ldgb_t *ldgb = ptr;

		/* maybe these two bits both determine the instruction encoding? */
		cat6->src_off = false;

596 597
		ldgb->d = instr->cat6.d - 1;
		ldgb->typed = instr->cat6.typed;
598 599 600 601 602 603 604 605 606 607 608 609 610 611
		ldgb->type_size = instr->cat6.iim_val - 1;

		ldgb->dst = reg(dst, info, instr->repeat, IR3_REG_R | IR3_REG_HALF);

		/* first src is src_ssbo: */
		iassert(src1->flags & IR3_REG_IMMED);
		ldgb->src_ssbo = src1->uim_val;

		/* then next two are src1/src2: */
		ldgb->src1 = reg(src2, info, instr->repeat, IR3_REG_IMMED);
		ldgb->src1_im = !!(src2->flags & IR3_REG_IMMED);
		ldgb->src2 = reg(src3, info, instr->repeat, IR3_REG_IMMED);
		ldgb->src2_im = !!(src3->flags & IR3_REG_IMMED);

612 613 614 615 616 617 618 619 620 621 622 623 624 625
		ldgb->pad0 = 0x0;
		ldgb->pad3 = 0x1;

		return 0;
	} else if (instr->opc == OPC_RESINFO) {
		instr_cat6ldgb_t *ldgb = ptr;

		ldgb->d = instr->cat6.d - 1;

		ldgb->dst = reg(dst, info, instr->repeat, IR3_REG_R | IR3_REG_HALF);

		/* first src is src_ssbo: */
		iassert(src1->flags & IR3_REG_IMMED);
		ldgb->src_ssbo = src1->uim_val;
626 627

		return 0;
628
	} else if ((instr->opc == OPC_STGB) || (instr->opc == OPC_STIB)) {
629 630 631 632 633 634 635
		struct ir3_register *src3 = instr->regs[4];
		instr_cat6stgb_t *stgb = ptr;

		/* maybe these two bits both determine the instruction encoding? */
		cat6->src_off = true;
		stgb->pad3 = 0x2;

636 637
		stgb->d = instr->cat6.d - 1;
		stgb->typed = instr->cat6.typed;
638 639 640 641 642 643 644 645 646 647 648 649 650 651
		stgb->type_size = instr->cat6.iim_val - 1;

		/* first src is dst_ssbo: */
		iassert(dst->flags & IR3_REG_IMMED);
		stgb->dst_ssbo = dst->uim_val;

		/* then src1/src2/src3: */
		stgb->src1 = reg(src1, info, instr->repeat, 0);
		stgb->src2 = reg(src2, info, instr->repeat, IR3_REG_IMMED);
		stgb->src2_im = !!(src2->flags & IR3_REG_IMMED);
		stgb->src3 = reg(src3, info, instr->repeat, IR3_REG_IMMED);
		stgb->src3_im = !!(src3->flags & IR3_REG_IMMED);

		return 0;
652 653
	} else if (instr->cat6.src_offset || (instr->opc == OPC_LDG) ||
			(instr->opc == OPC_LDL)) {
Rob Clark's avatar
Rob Clark committed
654 655
		instr_cat6a_t *cat6a = ptr;

656
		cat6->src_off = true;
Rob Clark's avatar
Rob Clark committed
657

658 659 660 661 662 663
		cat6a->src1 = reg(src1, info, instr->repeat, IR3_REG_IMMED);
		cat6a->src1_im = !!(src1->flags & IR3_REG_IMMED);
		if (src2) {
			cat6a->src2 = reg(src2, info, instr->repeat, IR3_REG_IMMED);
			cat6a->src2_im = !!(src2->flags & IR3_REG_IMMED);
		}
664
		cat6a->off = instr->cat6.src_offset;
665
	} else {
Rob Clark's avatar
Rob Clark committed
666 667
		instr_cat6b_t *cat6b = ptr;

668
		cat6->src_off = false;
Rob Clark's avatar
Rob Clark committed
669

670
		cat6b->src1 = reg(src1, info, instr->repeat, IR3_REG_IMMED | IR3_REG_HALF);
671 672 673 674 675
		cat6b->src1_im = !!(src1->flags & IR3_REG_IMMED);
		if (src2) {
			cat6b->src2 = reg(src2, info, instr->repeat, IR3_REG_IMMED);
			cat6b->src2_im = !!(src2->flags & IR3_REG_IMMED);
		}
Rob Clark's avatar
Rob Clark committed
676 677
	}

678 679
	if (instr->cat6.dst_offset || (instr->opc == OPC_STG) ||
			(instr->opc == OPC_STL)) {
680 681 682 683 684 685 686 687 688 689
		instr_cat6c_t *cat6c = ptr;
		cat6->dst_off = true;
		cat6c->dst = reg(dst, info, instr->repeat, IR3_REG_R | IR3_REG_HALF);
		cat6c->off = instr->cat6.dst_offset;
	} else {
		instr_cat6d_t *cat6d = ptr;
		cat6->dst_off = false;
		cat6d->dst = reg(dst, info, instr->repeat, IR3_REG_R | IR3_REG_HALF);
	}

Rob Clark's avatar
Rob Clark committed
690 691 692
	return 0;
}

693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710
static int emit_cat7(struct ir3_instruction *instr, void *ptr,
		struct ir3_info *info)
{
	instr_cat7_t *cat7 = ptr;

	cat7->ss      = !!(instr->flags & IR3_INSTR_SS);
	cat7->w       = instr->cat7.w;
	cat7->r       = instr->cat7.r;
	cat7->l       = instr->cat7.l;
	cat7->g       = instr->cat7.g;
	cat7->opc     = instr->opc;
	cat7->jmp_tgt = !!(instr->flags & IR3_INSTR_JP);
	cat7->sync    = !!(instr->flags & IR3_INSTR_SY);
	cat7->opc_cat = 7;

	return 0;
}

Rob Clark's avatar
Rob Clark committed
711
static int (*emit[])(struct ir3_instruction *instr, void *ptr,
712
		struct ir3_info *info) = {
Rob Clark's avatar
Rob Clark committed
713
	emit_cat0, emit_cat1, emit_cat2, emit_cat3, emit_cat4, emit_cat5, emit_cat6,
714
	emit_cat7,
Rob Clark's avatar
Rob Clark committed
715 716
};

717 718
void * ir3_assemble(struct ir3 *shader, struct ir3_info *info,
		uint32_t gpu_id)
Rob Clark's avatar
Rob Clark committed
719 720 721
{
	uint32_t *ptr, *dwords;

722
	info->gpu_id        = gpu_id;
Rob Clark's avatar
Rob Clark committed
723 724 725
	info->max_reg       = -1;
	info->max_half_reg  = -1;
	info->max_const     = -1;
726
	info->instrs_count  = 0;
727
	info->sizedwords    = 0;
728
	info->ss = info->sy = 0;
729

730 731 732 733
	list_for_each_entry (struct ir3_block, block, &shader->block_list, node) {
		list_for_each_entry (struct ir3_instruction, instr, &block->instr_list, node) {
			info->sizedwords += 2;
		}
734
	}
Rob Clark's avatar
Rob Clark committed
735

736
	/* need an integer number of instruction "groups" (sets of 16
737 738
	 * instructions on a4xx or sets of 4 instructions on a3xx),
	 * so pad out w/ NOPs if needed: (NOTE each instruction is 64bits)
Rob Clark's avatar
Rob Clark committed
739
	 */
740
	if (gpu_id >= 400) {
741
		info->sizedwords = align(info->sizedwords, 16 * 2);
742
	} else {
743
		info->sizedwords = align(info->sizedwords, 4 * 2);
744
	}
Rob Clark's avatar
Rob Clark committed
745

746
	ptr = dwords = calloc(4, info->sizedwords);
Rob Clark's avatar
Rob Clark committed
747

748 749
	list_for_each_entry (struct ir3_block, block, &shader->block_list, node) {
		list_for_each_entry (struct ir3_instruction, instr, &block->instr_list, node) {
750
			int ret = emit[opc_cat(instr->opc)](instr, dwords, info);
751 752 753 754
			if (ret)
				goto fail;
			info->instrs_count += 1 + instr->repeat;
			dwords += 2;
755 756 757 758 759 760

			if (instr->flags & IR3_INSTR_SS)
				info->ss++;

			if (instr->flags & IR3_INSTR_SY)
				info->sy++;
761
		}
Rob Clark's avatar
Rob Clark committed
762 763 764 765 766 767 768 769 770
	}

	return ptr;

fail:
	free(ptr);
	return NULL;
}

771
static struct ir3_register * reg_create(struct ir3 *shader,
Rob Clark's avatar
Rob Clark committed
772 773 774 775
		int num, int flags)
{
	struct ir3_register *reg =
			ir3_alloc(shader, sizeof(struct ir3_register));
776
	reg->wrmask = 1;
Rob Clark's avatar
Rob Clark committed
777 778 779 780 781
	reg->flags = flags;
	reg->num = num;
	return reg;
}

782
static void insert_instr(struct ir3_block *block,
Rob Clark's avatar
Rob Clark committed
783 784
		struct ir3_instruction *instr)
{
785
	struct ir3 *shader = block->shader;
786
#ifdef DEBUG
787
	instr->serialno = ++shader->instr_count;
788
#endif
789
	list_addtail(&instr->node, &block->instr_list);
790

791
	if (is_input(instr))
792
		array_insert(shader, shader->baryfs, instr);
Rob Clark's avatar
Rob Clark committed
793 794
}

795
struct ir3_block * ir3_block_create(struct ir3 *shader)
796
{
797
	struct ir3_block *block = ir3_alloc(shader, sizeof(*block));
798
#ifdef DEBUG
799
	block->serialno = ++shader->block_count;
800
#endif
801
	block->shader = shader;
802
	list_inithead(&block->node);
803
	list_inithead(&block->instr_list);
804 805 806
	return block;
}

807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824
static struct ir3_instruction *instr_create(struct ir3_block *block, int nreg)
{
	struct ir3_instruction *instr;
	unsigned sz = sizeof(*instr) + (nreg * sizeof(instr->regs[0]));
	char *ptr = ir3_alloc(block->shader, sz);

	instr = (struct ir3_instruction *)ptr;
	ptr  += sizeof(*instr);
	instr->regs = (struct ir3_register **)ptr;

#ifdef DEBUG
	instr->regs_max = nreg;
#endif

	return instr;
}

struct ir3_instruction * ir3_instr_create2(struct ir3_block *block,
825
		opc_t opc, int nreg)
Rob Clark's avatar
Rob Clark committed
826
{
827
	struct ir3_instruction *instr = instr_create(block, nreg);
828
	instr->block = block;
Rob Clark's avatar
Rob Clark committed
829
	instr->opc = opc;
830
	insert_instr(block, instr);
Rob Clark's avatar
Rob Clark committed
831 832 833
	return instr;
}

834
struct ir3_instruction * ir3_instr_create(struct ir3_block *block, opc_t opc)
835 836 837 838
{
	/* NOTE: we could be slightly more clever, at least for non-meta,
	 * and choose # of regs based on category.
	 */
839
	return ir3_instr_create2(block, opc, 4);
840 841
}

Rob Clark's avatar
Rob Clark committed
842 843
struct ir3_instruction * ir3_instr_clone(struct ir3_instruction *instr)
{
844 845 846
	struct ir3_instruction *new_instr = instr_create(instr->block,
			instr->regs_count);
	struct ir3_register **regs;
Rob Clark's avatar
Rob Clark committed
847 848
	unsigned i;

849
	regs = new_instr->regs;
Rob Clark's avatar
Rob Clark committed
850
	*new_instr = *instr;
851 852
	new_instr->regs = regs;

853
	insert_instr(instr->block, new_instr);
Rob Clark's avatar
Rob Clark committed
854 855 856 857 858 859 860 861 862 863 864 865 866

	/* clone registers: */
	new_instr->regs_count = 0;
	for (i = 0; i < instr->regs_count; i++) {
		struct ir3_register *reg = instr->regs[i];
		struct ir3_register *new_reg =
				ir3_reg_create(new_instr, reg->num, reg->flags);
		*new_reg = *reg;
	}

	return new_instr;
}

867 868 869 870 871 872
/* Add a false dependency to instruction, to ensure it is scheduled first: */
void ir3_instr_add_dep(struct ir3_instruction *instr, struct ir3_instruction *dep)
{
	array_insert(instr, instr->deps, dep);
}

Rob Clark's avatar
Rob Clark committed
873 874 875
struct ir3_register * ir3_reg_create(struct ir3_instruction *instr,
		int num, int flags)
{
876 877
	struct ir3 *shader = instr->block->shader;
	struct ir3_register *reg = reg_create(shader, num, flags);
878 879 880
#ifdef DEBUG
	debug_assert(instr->regs_count < instr->regs_max);
#endif
Rob Clark's avatar
Rob Clark committed
881 882 883
	instr->regs[instr->regs_count++] = reg;
	return reg;
}
884

Rob Clark's avatar
Rob Clark committed
885 886 887 888 889 890 891 892
struct ir3_register * ir3_reg_clone(struct ir3 *shader,
		struct ir3_register *reg)
{
	struct ir3_register *new_reg = reg_create(shader, 0, 0);
	*new_reg = *reg;
	return new_reg;
}

893 894 895 896 897 898 899
void
ir3_instr_set_address(struct ir3_instruction *instr,
		struct ir3_instruction *addr)
{
	if (instr->address != addr) {
		struct ir3 *ir = instr->block->shader;
		instr->address = addr;
900
		array_insert(ir, ir->indirects, instr);
901 902 903
	}
}

904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919
void
ir3_block_clear_mark(struct ir3_block *block)
{
	list_for_each_entry (struct ir3_instruction, instr, &block->instr_list, node)
		instr->flags &= ~IR3_INSTR_MARK;
}

void
ir3_clear_mark(struct ir3 *ir)
{
	list_for_each_entry (struct ir3_block, block, &ir->block_list, node) {
		ir3_block_clear_mark(block);
	}
}

/* note: this will destroy instr->depth, don't do it until after sched! */
920
unsigned
921 922
ir3_count_instructions(struct ir3 *ir)
{
923
	unsigned cnt = 0;
924 925
	list_for_each_entry (struct ir3_block, block, &ir->block_list, node) {
		list_for_each_entry (struct ir3_instruction, instr, &block->instr_list, node) {
926
			instr->ip = cnt++;
927 928 929 930
		}
		block->start_ip = list_first_entry(&block->instr_list, struct ir3_instruction, node)->ip;
		block->end_ip = list_last_entry(&block->instr_list, struct ir3_instruction, node)->ip;
	}
931
	return cnt;
932
}
Rob Clark's avatar
Rob Clark committed
933 934 935 936 937 938 939 940 941

struct ir3_array *
ir3_lookup_array(struct ir3 *ir, unsigned id)
{
	list_for_each_entry (struct ir3_array, arr, &ir->array_list, node)
		if (arr->id == id)
			return arr;
	return NULL;
}