Commit 077d01b6 authored by Eric Anholt's avatar Eric Anholt

i965: Add support for instruction compaction.

This reduces program size by using some smaller encodings for common bit
patterns in the Gen ISA, with the hope of making programs fit in the
instruction cache better.

v2: Use larger bitshifts for the uncompressed field setups, in line with the
    way it's described in the spec.  Consistently name a brw_compile "p" like
    all other code.  Add a couple more tests.  Consistently call things
    "compacted" not "compressed" (which is a different feature).  Drop the
    explicit check for not compacting SENDs, which is unjustified and already
    implied by our lack of support for immediate values.
Reviewed-by: Paul Berry's avatarPaul Berry <stereotype441@gmail.com>
parent f5e27063
Makefile
i965_symbols_test
libi965_dri.la
test_eu_compact
......@@ -51,16 +51,30 @@ libi965_dri_la_SOURCES = \
$(i965_C_FILES) \
$(i965_CXX_FILES)
# list of libs to be linked against by i965_dri.so and i965 test programs.
COMMON_LIBS = \
libi965_dri.la \
$(DRI_LIB_DEPS) \
$(INTEL_LIBS) \
../common/libdricommon.la
TEST_LIBS = \
$(COMMON_LIBS) \
-lrt \
../common/libdri_test_stubs.la
i965_dri_la_SOURCES =
i965_dri_la_LIBADD = $(COMMON_LIBS)
i965_dri_la_LDFLAGS = -module -avoid-version -shared
TESTS = test_eu_compact
check_PROGRAMS = test_eu_compact
test_eu_compact_SOURCES = \
test_eu_compact.c
nodist_EXTRA_test_eu_compact_SOURCES = dummy.cpp
test_eu_compact_LDADD = $(TEST_LIBS)
# Provide compatibility with scripts for the old Mesa build system for
# a while by putting a link to the driver into /lib of the build tree.
all-local: i965_dri.la
......
......@@ -44,6 +44,7 @@ i965_C_FILES = \
brw_draw.c \
brw_draw_upload.c \
brw_eu.c \
brw_eu_compact.c \
brw_eu_debug.c \
brw_eu_emit.c \
brw_eu_util.c \
......
......@@ -214,6 +214,11 @@ const GLuint *brw_get_program( struct brw_compile *p,
{
GLuint i;
brw_compact_instructions(p);
/* We emit a cacheline (8 instructions) of NOPs at the end of the program to
* make sure that instruction prefetch doesn't wander off into some other BO.
*/
for (i = 0; i < 8; i++)
brw_NOP(p);
......@@ -224,19 +229,36 @@ const GLuint *brw_get_program( struct brw_compile *p,
void
brw_dump_compile(struct brw_compile *p, FILE *out, int start, int end)
{
struct brw_context *brw = p->brw;
struct intel_context *intel = &brw->intel;
void *store = p->store;
bool dump_hex = false;
for (int offset = start; offset < end; offset += 16) {
for (int offset = start; offset < end;) {
struct brw_instruction *insn = store + offset;
struct brw_instruction uncompacted;
printf("0x%08x: ", offset);
if (0) {
printf("0x%08x 0x%08x 0x%08x 0x%08x ",
((uint32_t *)insn)[3],
((uint32_t *)insn)[2],
((uint32_t *)insn)[1],
((uint32_t *)insn)[0]);
if (insn->header.cmpt_control) {
struct brw_compact_instruction *compacted = (void *)insn;
if (dump_hex) {
printf("0x%08x 0x%08x ",
((uint32_t *)insn)[1],
((uint32_t *)insn)[0]);
}
brw_uncompact_instruction(intel, &uncompacted, compacted);
insn = &uncompacted;
offset += 8;
} else {
if (dump_hex) {
printf("0x%08x 0x%08x 0x%08x 0x%08x ",
((uint32_t *)insn)[3],
((uint32_t *)insn)[2],
((uint32_t *)insn)[1],
((uint32_t *)insn)[0]);
}
offset += 16;
}
brw_disasm(stdout, insn, p->brw->intel.gen);
......
......@@ -1107,6 +1107,19 @@ void brw_set_uip_jip(struct brw_compile *p);
uint32_t brw_swap_cmod(uint32_t cmod);
/* brw_eu_compact.c */
void brw_compact_instructions(struct brw_compile *p);
void brw_uncompact_instruction(struct intel_context *intel,
struct brw_instruction *dst,
struct brw_compact_instruction *src);
bool brw_try_compact_instruction(struct brw_compile *p,
struct brw_compact_instruction *dst,
struct brw_instruction *src);
void brw_debug_compact_uncompact(struct intel_context *intel,
struct brw_instruction *orig,
struct brw_instruction *uncompacted);
/* brw_optimize.c */
void brw_optimize(struct brw_compile *p);
void brw_remove_duplicate_mrf_moves(struct brw_compile *p);
......
/*
* Copyright © 2012 Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
/** @file brw_eu_compact.c
*
* Instruction compaction is a feature of gm45 and newer hardware that allows
* for a smaller instruction encoding.
*
* The instruction cache is on the order of 32KB, and many programs generate
* far more instructions than that. The instruction cache is built to barely
* keep up with instruction dispatch abaility in cache hit cases -- L1
* instruction cache misses that still hit in the next level could limit
* throughput by around 50%.
*
* The idea of instruction compaction is that most instructions use a tiny
* subset of the GPU functionality, so we can encode what would be a 16 byte
* instruction in 8 bytes using some lookup tables for various fields.
*/
#include "brw_context.h"
#include "brw_eu.h"
static const uint32_t gen6_control_index_table[32] = {
0b00000000000000000,
0b01000000000000000,
0b00110000000000000,
0b00000000100000000,
0b00010000000000000,
0b00001000100000000,
0b00000000100000010,
0b00000000000000010,
0b01000000100000000,
0b01010000000000000,
0b10110000000000000,
0b00100000000000000,
0b11010000000000000,
0b11000000000000000,
0b01001000100000000,
0b01000000000001000,
0b01000000000000100,
0b00000000000001000,
0b00000000000000100,
0b00111000100000000,
0b00001000100000010,
0b00110000100000000,
0b00110000000000001,
0b00100000000000001,
0b00110000000000010,
0b00110000000000101,
0b00110000000001001,
0b00110000000010000,
0b00110000000000011,
0b00110000000000100,
0b00110000100001000,
0b00100000000001001
};
static const uint32_t gen6_datatype_table[32] = {
0b001001110000000000,
0b001000110000100000,
0b001001110000000001,
0b001000000001100000,
0b001010110100101001,
0b001000000110101101,
0b001100011000101100,
0b001011110110101101,
0b001000000111101100,
0b001000000001100001,
0b001000110010100101,
0b001000000001000001,
0b001000001000110001,
0b001000001000101001,
0b001000000000100000,
0b001000001000110010,
0b001010010100101001,
0b001011010010100101,
0b001000000110100101,
0b001100011000101001,
0b001011011000101100,
0b001011010110100101,
0b001011110110100101,
0b001111011110111101,
0b001111011110111100,
0b001111011110111101,
0b001111011110011101,
0b001111011110111110,
0b001000000000100001,
0b001000000000100010,
0b001001111111011101,
0b001000001110111110,
};
static const uint32_t gen6_subreg_table[32] = {
0b000000000000000,
0b000000000000100,
0b000000110000000,
0b111000000000000,
0b011110000001000,
0b000010000000000,
0b000000000010000,
0b000110000001100,
0b001000000000000,
0b000001000000000,
0b000001010010100,
0b000000001010110,
0b010000000000000,
0b110000000000000,
0b000100000000000,
0b000000010000000,
0b000000000001000,
0b100000000000000,
0b000001010000000,
0b001010000000000,
0b001100000000000,
0b000000001010100,
0b101101010010100,
0b010100000000000,
0b000000010001111,
0b011000000000000,
0b111110000000000,
0b101000000000000,
0b000000000001111,
0b000100010001111,
0b001000010001111,
0b000110000000000,
};
static const uint32_t gen6_src_index_table[32] = {
0b000000000000,
0b010110001000,
0b010001101000,
0b001000101000,
0b011010010000,
0b000100100000,
0b010001101100,
0b010101110000,
0b011001111000,
0b001100101000,
0b010110001100,
0b001000100000,
0b010110001010,
0b000000000010,
0b010101010000,
0b010101101000,
0b111101001100,
0b111100101100,
0b011001110000,
0b010110001001,
0b010101011000,
0b001101001000,
0b010000101100,
0b010000000000,
0b001101110000,
0b001100010000,
0b001100000000,
0b010001101010,
0b001101111000,
0b000001110000,
0b001100100000,
0b001101010000,
};
static bool
set_control_index(struct brw_compact_instruction *dst,
struct brw_instruction *src)
{
uint32_t *src_u32 = (uint32_t *)src;
uint32_t uncompacted = 0;
uncompacted |= ((src_u32[0] >> 8) & 0xffff) << 0;
uncompacted |= ((src_u32[0] >> 31) & 0x1) << 16;
for (int i = 0; i < ARRAY_SIZE(gen6_control_index_table); i++) {
if (gen6_control_index_table[i] == uncompacted) {
dst->dw0.control_index = i;
return true;
}
}
return false;
}
static bool
set_datatype_index(struct brw_compact_instruction *dst,
struct brw_instruction *src)
{
uint32_t uncompacted = 0;
uncompacted |= src->bits1.ud & 0x7fff;
uncompacted |= (src->bits1.ud >> 29) << 15;
for (int i = 0; i < ARRAY_SIZE(gen6_datatype_table); i++) {
if (gen6_datatype_table[i] == uncompacted) {
dst->dw0.data_type_index = i;
return true;
}
}
return false;
}
static bool
set_subreg_index(struct brw_compact_instruction *dst,
struct brw_instruction *src)
{
uint32_t uncompacted = 0;
uncompacted |= src->bits1.da1.dest_subreg_nr << 0;
uncompacted |= src->bits2.da1.src0_subreg_nr << 5;
uncompacted |= src->bits3.da1.src1_subreg_nr << 10;
for (int i = 0; i < ARRAY_SIZE(gen6_subreg_table); i++) {
if (gen6_subreg_table[i] == uncompacted) {
dst->dw0.sub_reg_index = i;
return true;
}
}
return false;
}
static bool
get_src_index(uint32_t uncompacted,
uint32_t *compacted)
{
for (int i = 0; i < ARRAY_SIZE(gen6_src_index_table); i++) {
if (gen6_src_index_table[i] == uncompacted) {
*compacted = i;
return true;
}
}
return false;
}
static bool
set_src0_index(struct brw_compact_instruction *dst,
struct brw_instruction *src)
{
uint32_t compacted, uncompacted = 0;
uncompacted |= (src->bits2.ud >> 13) & 0xfff;
if (!get_src_index(uncompacted, &compacted))
return false;
dst->dw0.src0_index = compacted & 0x3;
dst->dw1.src0_index = compacted >> 2;
return true;
}
static bool
set_src1_index(struct brw_compact_instruction *dst,
struct brw_instruction *src)
{
uint32_t compacted, uncompacted = 0;
uncompacted |= (src->bits3.ud >> 13) & 0xfff;
if (!get_src_index(uncompacted, &compacted))
return false;
dst->dw1.src1_index = compacted;
return true;
}
/**
* Tries to compact instruction src into dst.
*
* It doesn't modify dst unless src is compactable, which is relied on by
* brw_compact_instructions().
*/
bool
brw_try_compact_instruction(struct brw_compile *p,
struct brw_compact_instruction *dst,
struct brw_instruction *src)
{
struct brw_compact_instruction temp;
/* FINISHME: immediates */
if (src->bits1.da1.src0_reg_file == BRW_IMMEDIATE_VALUE ||
src->bits1.da1.src1_reg_file == BRW_IMMEDIATE_VALUE)
return false;
memset(&temp, 0, sizeof(temp));
temp.dw0.opcode = src->header.opcode;
temp.dw0.debug_control = src->header.debug_control;
if (!set_control_index(&temp, src))
return false;
if (!set_datatype_index(&temp, src))
return false;
if (!set_subreg_index(&temp, src))
return false;
temp.dw0.acc_wr_control = src->header.acc_wr_control;
temp.dw0.conditionalmod = src->header.destreg__conditionalmod;
temp.dw0.flag_reg_nr = src->bits2.da1.flag_reg_nr;
temp.dw0.cmpt_ctrl = 1;
if (!set_src0_index(&temp, src))
return false;
if (!set_src1_index(&temp, src))
return false;
temp.dw1.dst_reg_nr = src->bits1.da1.dest_reg_nr;
temp.dw1.src0_reg_nr = src->bits2.da1.src0_reg_nr;
temp.dw1.src1_reg_nr = src->bits3.da1.src1_reg_nr;
*dst = temp;
return true;
}
static void
set_uncompacted_control(struct brw_instruction *dst,
struct brw_compact_instruction *src)
{
uint32_t *dst_u32 = (uint32_t *)dst;
uint32_t uncompacted = gen6_control_index_table[src->dw0.control_index];
dst_u32[0] |= ((uncompacted >> 0) & 0xffff) << 8;
dst_u32[0] |= ((uncompacted >> 16) & 0x1) << 31;
}
static void
set_uncompacted_datatype(struct brw_instruction *dst,
struct brw_compact_instruction *src)
{
uint32_t uncompacted = gen6_datatype_table[src->dw0.data_type_index];
dst->bits1.ud &= ~(0x7 << 29);
dst->bits1.ud |= ((uncompacted >> 15) & 0x7) << 29;
dst->bits1.ud &= ~0x7fff;
dst->bits1.ud |= uncompacted & 0x7fff;
}
static void
set_uncompacted_subreg(struct brw_instruction *dst,
struct brw_compact_instruction *src)
{
uint32_t uncompacted = gen6_subreg_table[src->dw0.sub_reg_index];
dst->bits1.da1.dest_subreg_nr = (uncompacted >> 0) & 0x1f;
dst->bits2.da1.src0_subreg_nr = (uncompacted >> 5) & 0x1f;
dst->bits3.da1.src1_subreg_nr = (uncompacted >> 10) & 0x1f;
}
static void
set_uncompacted_src0(struct brw_instruction *dst,
struct brw_compact_instruction *src)
{
uint32_t compacted = src->dw0.src0_index | src->dw1.src0_index << 2;
uint32_t uncompacted = gen6_src_index_table[compacted];
dst->bits2.ud |= uncompacted << 13;
}
static void
set_uncompacted_src1(struct brw_instruction *dst,
struct brw_compact_instruction *src)
{
uint32_t uncompacted = gen6_src_index_table[src->dw1.src1_index];
dst->bits3.ud |= uncompacted << 13;
}
void
brw_uncompact_instruction(struct intel_context *intel,
struct brw_instruction *dst,
struct brw_compact_instruction *src)
{
memset(dst, 0, sizeof(*dst));
dst->header.opcode = src->dw0.opcode;
dst->header.debug_control = src->dw0.debug_control;
set_uncompacted_control(dst, src);
set_uncompacted_datatype(dst, src);
set_uncompacted_subreg(dst, src);
dst->header.acc_wr_control = src->dw0.acc_wr_control;
dst->header.destreg__conditionalmod = src->dw0.conditionalmod;
dst->bits2.da1.flag_reg_nr = src->dw0.flag_reg_nr;
set_uncompacted_src0(dst, src);
set_uncompacted_src1(dst, src);
dst->bits1.da1.dest_reg_nr = src->dw1.dst_reg_nr;
dst->bits2.da1.src0_reg_nr = src->dw1.src0_reg_nr;
dst->bits3.da1.src1_reg_nr = src->dw1.src1_reg_nr;
}
void brw_debug_compact_uncompact(struct intel_context *intel,
struct brw_instruction *orig,
struct brw_instruction *uncompacted)
{
fprintf(stderr, "Instruction compact/uncompact changed:\n");
fprintf(stderr, " before: ");
brw_disasm(stderr, orig, intel->gen);
fprintf(stderr, " after: ");
brw_disasm(stderr, uncompacted, intel->gen);
uint32_t *before_bits = (uint32_t *)orig;
uint32_t *after_bits = (uint32_t *)uncompacted;
printf(" changed bits:\n");
for (int i = 0; i < 128; i++) {
uint32_t before = before_bits[i / 32] & (1 << (i & 31));
uint32_t after = after_bits[i / 32] & (1 << (i & 31));
if (before != after) {
printf(" bit %d, %s to %s\n", i,
before ? "set" : "unset",
after ? "set" : "unset");
}
}
}
void
brw_compact_instructions(struct brw_compile *p)
{
struct brw_context *brw = p->brw;
struct intel_context *intel = &brw->intel;
void *store = p->store;
assert(gen6_control_index_table[ARRAY_SIZE(gen6_control_index_table) - 1] != 0);
assert(gen6_datatype_table[ARRAY_SIZE(gen6_datatype_table) - 1] != 0);
assert(gen6_subreg_table[ARRAY_SIZE(gen6_subreg_table) - 1] != 0);
assert(gen6_src_index_table[ARRAY_SIZE(gen6_src_index_table) - 1] != 0);
if (intel->gen != 6)
return;
/* FINISHME: If we are going to compress instructions between flow control,
* we have to do fixups to flow control offsets to represent the new
* distances, since flow control uses (virtual address distance)/2, not a
* logical instruction count. We can at least compress up until an IF
* instruction, but there's no instruction indicating the start of a
* do/while loop.
*/
bool continue_compressing = true;
for (int i = 0; i < p->nr_insn; i++) {
if (p->store[i].header.opcode == BRW_OPCODE_WHILE)
return;
}
int src_offset;
int offset = 0;
for (src_offset = 0; src_offset < p->nr_insn * 16;) {
struct brw_instruction *src = store + src_offset;
void *dst = store + offset;
switch (src->header.opcode) {
case BRW_OPCODE_IF:
case BRW_OPCODE_HALT:
case BRW_OPCODE_JMPI:
continue_compressing = false;
break;
}
struct brw_instruction saved = *src;
if (continue_compressing &&
!src->header.cmpt_control &&
brw_try_compact_instruction(p, dst, src)) {
/* debug */
if (INTEL_DEBUG) {
struct brw_instruction uncompacted;
brw_uncompact_instruction(intel, &uncompacted, dst);
if (memcmp(&saved, &uncompacted, sizeof(uncompacted))) {
brw_debug_compact_uncompact(intel, &saved, &uncompacted);
}
}
offset += 8;
src_offset += 16;
} else {
int size = src->header.cmpt_control ? 8 : 16;
/* It appears that the end of thread SEND instruction needs to be
* aligned, or the GPU hangs.
*/
if ((src->header.opcode == BRW_OPCODE_SEND ||
src->header.opcode == BRW_OPCODE_SENDC) &&