bi_pack.c 39.7 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
/*
 * Copyright (C) 2020 Collabora, Ltd.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include "compiler.h"

Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
26
27
28
29
30
31
#define RETURN_PACKED(str) { \
        uint64_t temp = 0; \
        memcpy(&temp, &str, sizeof(str)); \
        return temp; \
}

32
33
34
35
36
/* This file contains the final passes of the compiler. Running after
 * scheduling and RA, the IR is now finalized, so we need to emit it to actual
 * bits on the wire (as well as fixup branches) */

static uint64_t
37
bi_pack_header(bi_clause *clause, bi_clause *next, bool is_fragment)
38
39
{
        struct bifrost_header header = {
40
                .back_to_back = clause->back_to_back,
41
                .no_end_of_shader = (next != NULL),
42
43
44
45
                .elide_writes = is_fragment,
                .branch_cond = clause->branch_conditional,
                .datareg_writebarrier = clause->data_register_write_barrier,
                .datareg = clause->data_register,
46
                .scoreboard_deps = next ? next->dependencies : 0,
47
48
49
                .scoreboard_index = clause->scoreboard_id,
                .clause_type = clause->clause_type,
                .next_clause_type = next ? next->clause_type : 0,
50
51
52
53
54
55
56
        };

        uint64_t u = 0;
        memcpy(&u, &header, sizeof(header));
        return u;
}

57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
/* Represents the assignment of ports for a given bundle */

struct bi_registers {
        /* Register to assign to each port */
        unsigned port[4];

        /* Read ports can be disabled */
        bool enabled[2];

        /* Should we write FMA? what about ADD? If only a single port is
         * enabled it is in port 2, else ADD/FMA is 2/3 respectively */
        bool write_fma, write_add;

        /* Should we read with port 3? */
        bool read_port3;

        /* Packed uniform/constant */
74
        uint8_t uniform_constant;
75
76
77
78
79

        /* Whether writes are actually for the last instruction */
        bool first_instruction;
};

80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
static inline void
bi_print_ports(struct bi_registers *regs)
{
        for (unsigned i = 0; i < 2; ++i) {
                if (regs->enabled[i])
                        printf("port %u: %u\n", i, regs->port[i]);
        }

        if (regs->write_fma || regs->write_add) {
                printf("port 2 (%s): %u\n",
                                regs->write_fma ? "FMA" : "ADD",
                                regs->port[2]);
        }

        if ((regs->write_fma && regs->write_add) || regs->read_port3) {
                printf("port 3 (%s): %u\n",
                                regs->read_port3 ? "read" : "ADD",
                                regs->port[3]);
        }
}

101
102
103
104
105
106
107
/* The uniform/constant slot allows loading a contiguous 64-bit immediate or
 * pushed uniform per bundle. Figure out which one we need in the bundle (the
 * scheduler needs to ensure we only have one type per bundle), validate
 * everything, and rewrite away the register/uniform indices to use 3-bit
 * sources directly. */

static unsigned
108
bi_lookup_constant(bi_clause *clause, uint64_t cons, bool *hi, bool b64)
109
{
110
111
        uint64_t want = (cons >> 4);

112
113
114
115
        for (unsigned i = 0; i < clause->constant_count; ++i) {
                /* Only check top 60-bits since that's what's actually embedded
                 * in the clause, the bottom 4-bits are bundle-inline */

116
117
118
119
120
121
122
123
124
125
126
127
128
                unsigned candidates[2] = {
                        clause->constants[i] >> 4,
                        clause->constants[i] >> 36
                };

                if (!b64)
                        candidates[0] &= 0xFFFFFFFF;

                if (candidates[0] == want)
                        return i;

                if (candidates[1] == want && !b64) {
                        *hi = true;
129
                        return i;
130
                }
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
        }

        unreachable("Invalid constant accessed");
}

static unsigned
bi_constant_field(unsigned idx)
{
        assert(idx <= 5);

        const unsigned values[] = {
                4, 5, 6, 7, 2, 3
        };

        return values[idx] << 4;
}

static bool
bi_assign_uniform_constant_single(
                struct bi_registers *regs,
                bi_clause *clause,
                bi_instruction *ins, bool assigned, bool fast_zero)
{
        if (!ins)
                return assigned;

        bi_foreach_src(ins, s) {
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
158
                if (s == 0 && (ins->type == BI_LOAD_VAR_ADDRESS || ins->type == BI_LOAD_ATTR)) continue;
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
159

160
                if (ins->src[s] & BIR_INDEX_CONSTANT) {
161
162
163
164
                        bool hi = false;
                        bool b64 = nir_alu_type_get_type_size(ins->src_types[s]) > 32;
                        uint64_t cons = bi_get_immediate(ins, ins->src[s]);
                        unsigned idx = bi_lookup_constant(clause, cons, &hi, b64);
165
166
167
168
169
170
                        unsigned f = bi_constant_field(idx) | (cons & 0xF);

                        if (assigned && regs->uniform_constant != f)
                                unreachable("Mismatched uniform/const field: imm");

                        regs->uniform_constant = f;
171
                        ins->src[s] = BIR_INDEX_PASS | (hi ? BIFROST_SRC_CONST_HI : BIFROST_SRC_CONST_LO);
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
                        assigned = true;
                } else if (ins->src[s] & BIR_INDEX_ZERO && (ins->type == BI_LOAD_UNIFORM || ins->type == BI_LOAD_VAR)) {
                        /* XXX: HACK UNTIL WE HAVE HI MATCHING DUE TO OVERFLOW XXX */
                        ins->src[s] = BIR_INDEX_PASS | BIFROST_SRC_CONST_HI;
                } else if (ins->src[s] & BIR_INDEX_ZERO && !fast_zero) {
                        /* FMAs have a fast zero port, ADD needs to use the
                         * uniform/const port's special 0 mode handled here */
                        unsigned f = 0;

                        if (assigned && regs->uniform_constant != f)
                                unreachable("Mismatched uniform/const field: 0");

                        regs->uniform_constant = f;
                        ins->src[s] = BIR_INDEX_PASS | BIFROST_SRC_CONST_LO;
                        assigned = true;
                } else if (s & BIR_INDEX_UNIFORM) {
                        unreachable("Push uniforms not implemented yet");
                }
        }

        return assigned;
}

static void
bi_assign_uniform_constant(
                bi_clause *clause,
                struct bi_registers *regs,
                bi_bundle bundle)
{
        bool assigned =
                bi_assign_uniform_constant_single(regs, clause, bundle.fma, false, true);

        bi_assign_uniform_constant_single(regs, clause, bundle.add, assigned, false);
}

207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
/* Assigns a port for reading, before anything is written */

static void
bi_assign_port_read(struct bi_registers *regs, unsigned src)
{
        /* We only assign for registers */
        if (!(src & BIR_INDEX_REGISTER))
                return;

        unsigned reg = src & ~BIR_INDEX_REGISTER;

        /* Check if we already assigned the port */
        for (unsigned i = 0; i <= 1; ++i) {
                if (regs->port[i] == reg && regs->enabled[i])
                        return;
        }

        if (regs->port[3] == reg && regs->read_port3)
                return;

        /* Assign it now */

        for (unsigned i = 0; i <= 1; ++i) {
                if (!regs->enabled[i]) {
                        regs->port[i] = reg;
                        regs->enabled[i] = true;
                        return;
                }
        }

        if (!regs->read_port3) {
                regs->port[3] = reg;
                regs->read_port3 = true;
240
                return;
241
        }
242
243
244

        bi_print_ports(regs);
        unreachable("Failed to find a free port for src");
245
246
247
248
249
250
251
}

static struct bi_registers
bi_assign_ports(bi_bundle now, bi_bundle prev)
{
        struct bi_registers regs = { 0 };

252
253
254
255
256
257
258
259
260
261
        /* We assign ports for the main register mechanism. Special ops
         * use the data registers, which has its own mechanism entirely
         * and thus gets skipped over here. */

        unsigned read_dreg = now.add &&
                bi_class_props[now.add->type] & BI_DATA_REG_SRC;

        unsigned write_dreg = prev.add &&
                bi_class_props[prev.add->type] & BI_DATA_REG_DEST;

262
263
264
265
266
267
        /* First, assign reads */

        if (now.fma)
                bi_foreach_src(now.fma, src)
                        bi_assign_port_read(&regs, now.fma->src[src]);

268
269
270
271
272
273
        if (now.add) {
                bi_foreach_src(now.add, src) {
                        if (!(src == 0 && read_dreg))
                                bi_assign_port_read(&regs, now.add->src[src]);
                }
        }
274
275
276
277
278
279
280
281

        /* Next, assign writes */

        if (prev.fma && prev.fma->dest & BIR_INDEX_REGISTER) {
                regs.port[2] = prev.fma->dest & ~BIR_INDEX_REGISTER;
                regs.write_fma = true;
        }

282
        if (prev.add && prev.add->dest & BIR_INDEX_REGISTER && !write_dreg) {
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
                unsigned r = prev.add->dest & ~BIR_INDEX_REGISTER;

                if (regs.write_fma) {
                        /* Scheduler constraint: cannot read 3 and write 2 */
                        assert(!regs.read_port3);
                        regs.port[3] = r;
                } else {
                        regs.port[2] = r;
                }

                regs.write_add = true;
        }

        /* Finally, ensure port 1 > port 0 for the 63-x trick to function */

        if (regs.enabled[0] && regs.enabled[1] && regs.port[1] < regs.port[0]) {
                unsigned temp = regs.port[0];
                regs.port[0] = regs.port[1];
                regs.port[1] = temp;
        }

        return regs;
}

307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
/* Determines the register control field, ignoring the first? flag */

static enum bifrost_reg_control
bi_pack_register_ctrl_lo(struct bi_registers r)
{
        if (r.write_fma) {
                if (r.write_add) {
                        assert(!r.read_port3);
                        return BIFROST_WRITE_ADD_P2_FMA_P3;
                } else {
                        if (r.read_port3)
                                return BIFROST_WRITE_FMA_P2_READ_P3;
                        else
                                return BIFROST_WRITE_FMA_P2;
                }
        } else if (r.write_add) {
                if (r.read_port3)
                        return BIFROST_WRITE_ADD_P2_READ_P3;
                else
                        return BIFROST_WRITE_ADD_P2;
        } else if (r.read_port3)
                return BIFROST_READ_P3;
        else
                return BIFROST_REG_NONE;
}

/* Ditto but account for the first? flag this time */

static enum bifrost_reg_control
bi_pack_register_ctrl(struct bi_registers r)
{
        enum bifrost_reg_control ctrl = bi_pack_register_ctrl_lo(r);

        if (r.first_instruction) {
                if (ctrl == BIFROST_REG_NONE)
                        ctrl = BIFROST_FIRST_NONE;
343
344
                else if (ctrl == BIFROST_WRITE_FMA_P2_READ_P3)
                        ctrl = BIFROST_FIRST_WRITE_FMA_P2_READ_P3;
345
346
347
348
349
350
351
                else
                        ctrl |= BIFROST_FIRST_NONE;
        }

        return ctrl;
}

352
static uint64_t
353
bi_pack_registers(struct bi_registers regs)
354
{
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
        enum bifrost_reg_control ctrl = bi_pack_register_ctrl(regs);
        struct bifrost_regs s;
        uint64_t packed = 0;

        if (regs.enabled[1]) {
                /* Gotta save that bit!~ Required by the 63-x trick */
                assert(regs.port[1] > regs.port[0]);
                assert(regs.enabled[0]);

                /* Do the 63-x trick, see docs/disasm */
                if (regs.port[0] > 31) {
                        regs.port[0] = 63 - regs.port[0];
                        regs.port[1] = 63 - regs.port[1];
                }

                assert(regs.port[0] <= 31);
                assert(regs.port[1] <= 63);

                s.ctrl = ctrl;
                s.reg1 = regs.port[1];
                s.reg0 = regs.port[0];
        } else {
                /* Port 1 disabled, so set to zero and use port 1 for ctrl */
                s.reg1 = ctrl << 2;

                if (regs.enabled[0]) {
                        /* Bit 0 upper bit of port 0 */
                        s.reg1 |= (regs.port[0] >> 5);

                        /* Rest of port 0 in usual spot */
                        s.reg0 = (regs.port[0] & 0b11111);
                } else {
                        /* Bit 1 set if port 0 also disabled */
                        s.reg1 |= (1 << 1);
                }
        }

392
        /* When port 3 isn't used, we have to set it to port 2, and vice versa,
393
         * or INSTR_INVALID_ENC is raised. The reason is unknown. */
394

395
396
397
398
        bool has_port2 = regs.write_fma || regs.write_add;
        bool has_port3 = regs.read_port3 || (regs.write_fma && regs.write_add);

        if (!has_port3)
399
400
                regs.port[3] = regs.port[2];

401
        if (!has_port2)
402
403
                regs.port[2] = regs.port[3];

404
405
406
407
408
409
        s.reg3 = regs.port[3];
        s.reg2 = regs.port[2];
        s.uniform_const = regs.uniform_constant;

        memcpy(&packed, &s, sizeof(s));
        return packed;
410
411
}

Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
412
static void
413
bi_set_data_register(bi_clause *clause, unsigned idx)
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
414
{
415
416
        assert(idx & BIR_INDEX_REGISTER);
        unsigned reg = idx & ~BIR_INDEX_REGISTER;
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
417
418
419
420
        assert(reg <= 63);
        clause->data_register = reg;
}

421
422
423
424
425
426
427
428
429
430
431
432
static void
bi_read_data_register(bi_clause *clause, bi_instruction *ins)
{
        bi_set_data_register(clause, ins->src[0]);
}

static void
bi_write_data_register(bi_clause *clause, bi_instruction *ins)
{
        bi_set_data_register(clause, ins->dest);
}

Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
static enum bifrost_packed_src
bi_get_src_reg_port(struct bi_registers *regs, unsigned src)
{
        unsigned reg = src & ~BIR_INDEX_REGISTER;

        if (regs->port[0] == reg && regs->enabled[0])
                return BIFROST_SRC_PORT0;
        else if (regs->port[1] == reg && regs->enabled[1])
                return BIFROST_SRC_PORT1;
        else if (regs->port[3] == reg && regs->read_port3)
                return BIFROST_SRC_PORT3;
        else
                unreachable("Tried to access register with no port");
}

448
449
static enum bifrost_packed_src
bi_get_src(bi_instruction *ins, struct bi_registers *regs, unsigned s, bool is_fma)
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
450
451
452
453
454
{
        unsigned src = ins->src[s];

        if (src & BIR_INDEX_REGISTER)
                return bi_get_src_reg_port(regs, src);
455
        else if (src & BIR_INDEX_ZERO && is_fma)
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
456
457
458
459
                return BIFROST_SRC_STAGE;
        else if (src & BIR_INDEX_PASS)
                return src & ~BIR_INDEX_PASS;
        else
460
                unreachable("Unknown src");
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
461
462
}

Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
/* Constructs a packed 2-bit swizzle for a 16-bit vec2 source. Source must be
 * 16-bit and written components must correspond to valid swizzles (component x
 * or y). */

static unsigned
bi_swiz16(bi_instruction *ins, unsigned src)
{
        assert(nir_alu_type_get_type_size(ins->src_types[src]) == 16);
        unsigned swizzle = 0;

        for (unsigned c = 0; c < 2; ++c) {
                if (!bi_writes_component(ins, src)) continue;

                unsigned k = ins->swizzle[src][c];
                assert(k < 1);
                swizzle |= (k << c);
        }

        return swizzle;
}

484
static unsigned
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
485
bi_pack_fma_fma(bi_instruction *ins, struct bi_registers *regs)
486
{
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
487
488
489
        /* (-a)(-b) = ab, so we only need one negate bit */
        bool negate_mul = ins->src_neg[0] ^ ins->src_neg[1];

Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
490
491
492
493
494
495
496
497
498
499
        if (ins->dest_type == nir_type_float32) {
                struct bifrost_fma_fma pack = {
                        .src0 = bi_get_src(ins, regs, 0, true),
                        .src1 = bi_get_src(ins, regs, 1, true),
                        .src2 = bi_get_src(ins, regs, 2, true),
                        .src0_abs = ins->src_abs[0],
                        .src1_abs = ins->src_abs[1],
                        .src2_abs = ins->src_abs[2],
                        .src0_neg = negate_mul,
                        .src2_neg = ins->src_neg[2],
500
501
                        .outmod = ins->outmod,
                        .roundmode = ins->roundmode,
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
502
503
                        .op = BIFROST_FMA_OP_FMA
                };
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
504

Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
505
506
507
508
509
510
511
512
513
514
515
                RETURN_PACKED(pack);
        } else if (ins->dest_type == nir_type_float16) {
                struct bifrost_fma_fma16 pack = {
                        .src0 = bi_get_src(ins, regs, 0, true),
                        .src1 = bi_get_src(ins, regs, 1, true),
                        .src2 = bi_get_src(ins, regs, 2, true),
                        .swizzle_0 = bi_swiz16(ins, 0),
                        .swizzle_1 = bi_swiz16(ins, 1),
                        .swizzle_2 = bi_swiz16(ins, 2),
                        .src0_neg = negate_mul,
                        .src2_neg = ins->src_neg[2],
516
517
                        .outmod = ins->outmod,
                        .roundmode = ins->roundmode,
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
518
519
520
521
522
523
524
                        .op = BIFROST_FMA_OP_FMA16
                };

                RETURN_PACKED(pack);
        } else {
                unreachable("Invalid fma dest type");
        }
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
525
526
}

Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
527
static unsigned
528
bi_pack_fma_addmin_f32(bi_instruction *ins, struct bi_registers *regs)
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
529
{
530
531
532
533
534
        unsigned op =
                (ins->type == BI_ADD) ? BIFROST_FMA_OP_FADD32 :
                (ins->op.minmax == BI_MINMAX_MIN) ? BIFROST_FMA_OP_FMIN32 :
                BIFROST_FMA_OP_FMAX32;

Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
535
        struct bifrost_fma_add pack = {
536
537
                .src0 = bi_get_src(ins, regs, 0, true),
                .src1 = bi_get_src(ins, regs, 1, true),
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
538
539
540
541
542
543
                .src0_abs = ins->src_abs[0],
                .src1_abs = ins->src_abs[1],
                .src0_neg = ins->src_neg[0],
                .src1_neg = ins->src_neg[1],
                .unk = 0x0,
                .outmod = ins->outmod,
544
545
                .roundmode = (ins->type == BI_ADD) ? ins->roundmode : ins->minmax,
                .op = op
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
546
547
548
549
550
        };

        RETURN_PACKED(pack);
}

551
552
553
554
555
556
557
558
static unsigned
bi_pack_fma_addmin_f16(bi_instruction *ins, struct bi_registers *regs)
{
        unsigned op =
                (ins->type == BI_ADD) ? BIFROST_FMA_OP_FADD16 :
                (ins->op.minmax == BI_MINMAX_MIN) ? BIFROST_FMA_OP_FMIN16 :
                BIFROST_FMA_OP_FMAX16;

559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
        /* Absolute values are packed in a quirky way. Let k = src1 < src0. Let
         * l be an auxiliary bit we encode. Then the hardware determines:
         *
         *      abs0 = l || k
         *      abs1 = l && k
         *
         * Since add/min/max are commutative, this saves a bit by using the
         * order of the operands as a bit (k). To pack this, first note:
         *
         *      (l && k) implies (l || k).
         *
         * That is, if the second argument is abs'd, then the first argument
         * also has abs. So there are three cases:
         *
         * Case 0: Neither src has absolute value. Then we have l = k = 0.
         *
         * Case 1: Exactly one src has absolute value. Assign that source to
         * src0 and the other source to src1. Compute k = src1 < src0 based on
         * that assignment. Then l = ~k.
         *
         * Case 2: Both sources have absolute value. Then we have l = k = 1.
         * Note to force k = 1 requires that (src1 < src0) OR (src0 < src1).
         * That is, this encoding is only valid if src1 and src0 are distinct.
         * This is a scheduling restriction (XXX); if an op of this type
         * requires both identical sources to have abs value, then we must
         * schedule to ADD (which does not use this ordering trick).
         */

        unsigned abs_0 = ins->src_abs[0], abs_1 = ins->src_abs[1];
        unsigned src_0 = bi_get_src(ins, regs, 0, true);
589
        unsigned src_1 = bi_get_src(ins, regs, 1, true);
590
        bool l = false;
591
        bool flip = false;
592
593
594

        if (!abs_0 && !abs_1) {
                /* Force k = 0 <===> NOT(src1 < src0) <==> src1 >= src0 */
595
                flip = (src_0 < src_1);
596
597
598
        } else if (abs_0 && !abs_1) {
                l = src_1 >= src_0;
        } else if (abs_1 && !abs_0) {
599
600
                flip = true;
                l = src_0 >= src_1;
601
        } else {
602
                flip = (src_0 >= src_1);
603
604
605
                l = true;
        }

606
        struct bifrost_fma_add_minmax16 pack = {
607
608
609
610
                .src0 = flip ? src_1 : src_0,
                .src1 = flip ? src_0 : src_1,
                .src0_neg = ins->src_neg[flip ? 1 : 0],
                .src1_neg = ins->src_neg[flip ? 0 : 1],
611
                .abs1 = l,
612
613
614
615
616
617
618
619
620
                .outmod = ins->outmod,
                .mode = (ins->type == BI_ADD) ? ins->roundmode : ins->minmax,
                .op = op
        };

        RETURN_PACKED(pack);
}

static unsigned
621
bi_pack_fma_addmin(bi_instruction *ins, struct bi_registers *regs)
622
623
{
        if (ins->dest_type == nir_type_float32)
624
                return bi_pack_fma_addmin_f32(ins, regs);
625
626
627
628
629
630
        else if(ins->dest_type == nir_type_float16)
                return bi_pack_fma_addmin_f16(ins, regs);
        else
                unreachable("Unknown FMA/ADD type");
}

631
632
633
634
635
636
637
638
639
640
641
static unsigned
bi_pack_fma_1src(bi_instruction *ins, struct bi_registers *regs, unsigned op)
{
        struct bifrost_fma_inst pack = {
                .src0 = bi_get_src(ins, regs, 0, true),
                .op = op
        };

        RETURN_PACKED(pack);
}

642
643
644
645
646
647
648
649
650
651
652
static unsigned
bi_pack_add_1src(bi_instruction *ins, struct bi_registers *regs, unsigned op)
{
        struct bifrost_add_inst pack = {
                .src0 = bi_get_src(ins, regs, 0, true),
                .op = op
        };

        RETURN_PACKED(pack);
}

Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
static enum bifrost_csel_cond
bi_cond_to_csel(enum bi_cond cond, bool *flip, bool *invert, nir_alu_type T)
{
        nir_alu_type B = nir_alu_type_get_base_type(T);
        unsigned idx = (B == nir_type_float) ? 0 :
                ((B == nir_type_int) ? 1 : 2);

        switch (cond){
        case BI_COND_LT:
                *flip = true;
        case BI_COND_GT: {
                const enum bifrost_csel_cond ops[] = {
                        BIFROST_FGT_F,
                        BIFROST_IGT_I,
                        BIFROST_UGT_I
                };

                return ops[idx];
        }
        case BI_COND_LE:
                *flip = true;
        case BI_COND_GE: {
                const enum bifrost_csel_cond ops[] = {
                        BIFROST_FGE_F,
                        BIFROST_IGE_I,
                        BIFROST_UGE_I
                };

                return ops[idx];
        }
        case BI_COND_NE:
                *invert = true;
        case BI_COND_EQ: {
                const enum bifrost_csel_cond ops[] = {
                        BIFROST_FEQ_F,
                        BIFROST_IEQ_F,
                        BIFROST_IEQ_F /* sign is irrelevant */
                };

                return ops[idx];
        }
        default:
                unreachable("Invalid op for csel");
        }
}

static unsigned
bi_pack_fma_csel(bi_instruction *ins, struct bi_registers *regs)
{
        /* TODO: Use csel3 as well */
        bool flip = false, invert = false;

        enum bifrost_csel_cond cond =
                bi_cond_to_csel(ins->csel_cond, &flip, &invert, ins->src_types[0]);

        unsigned size = nir_alu_type_get_type_size(ins->dest_type);

710
711
712
713
        unsigned cmp_0 = (flip ? 1 : 0);
        unsigned cmp_1 = (flip ? 0 : 1);
        unsigned res_0 = (invert ? 3 : 2);
        unsigned res_1 = (invert ? 2 : 3);
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
714
715
716
717
718
719
720
721
722
723
724
725
726
727
        
        struct bifrost_csel4 pack = {
                .src0 = bi_get_src(ins, regs, cmp_0, true),
                .src1 = bi_get_src(ins, regs, cmp_1, true),
                .src2 = bi_get_src(ins, regs, res_0, true),
                .src3 = bi_get_src(ins, regs, res_1, true),
                .cond = cond,
                .op = (size == 16) ? BIFROST_FMA_OP_CSEL4_V16 :
                        BIFROST_FMA_OP_CSEL4
        };

        RETURN_PACKED(pack);
}

728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
/* We have a single convert opcode in the IR but a number of opcodes that could
 * come out. In particular we have native opcodes for:
 *
 * [ui]16 --> [fui]32           -- int16_to_32
 * f16     --> f32              -- float16_to_32
 * f32     --> f16              -- float32_to_16
 * f32     --> [ui]32           -- float32_to_int
 * [ui]32  --> f32              -- int_to_float32
 * [fui]16 --> [fui]16          -- f2i_i2f16
 */

static unsigned
bi_pack_fma_convert(bi_instruction *ins, struct bi_registers *regs)
{
        nir_alu_type from_base = nir_alu_type_get_base_type(ins->src_types[0]);
        unsigned from_size = nir_alu_type_get_type_size(ins->src_types[0]);
744
        bool from_unsigned = from_base == nir_type_uint;
745
746
747

        nir_alu_type to_base = nir_alu_type_get_base_type(ins->dest_type);
        unsigned to_size = nir_alu_type_get_type_size(ins->dest_type);
748
        bool to_unsigned = to_base == nir_type_uint;
749
750
751
752
753
754
755

        /* Sanity check */
        assert((from_base != to_base) || (from_size != to_size));
        assert((MAX2(from_size, to_size) / MIN2(from_size, to_size)) <= 2);

        if (from_size == 16 && to_size == 16) {
                /* f2i_i2f16 */
756
                unreachable("i16 not yet implemented");
757
        } else if (from_size == 32 && to_size == 32) {
758
759
                unsigned op = 0;

760
                if (from_base == nir_type_float) {
761
                        op = BIFROST_FMA_FLOAT32_TO_INT(to_unsigned);
762
                } else {
763
                        op = BIFROST_FMA_INT_TO_FLOAT32(from_unsigned);
764
                }
765
766

                return bi_pack_fma_1src(ins, regs, op);
767
        } else if (from_size == 16 && to_size == 32) {
768
769
                bool from_y = ins->swizzle[0][0];

770
                if (from_base == nir_type_float) {
771
772
                        return bi_pack_fma_1src(ins, regs,
                                        BIFROST_FMA_FLOAT16_TO_32(from_y));
773
                } else {
774
                        unreachable("i16 not yet implemented");
775
776
777
                }
        } else if (from_size == 32 && to_size == 16) {
                if (from_base == nir_type_float) {
778
779
780
781
782
783
784
785
                        /* TODO: second vectorized source? */
                        struct bifrost_fma_2src pack = {
                                .src0 = bi_get_src(ins, regs, 0, true),
                                .src1 = BIFROST_SRC_STAGE, /* 0 */
                                .op = BIFROST_FMA_FLOAT32_TO_16
                        };

                        RETURN_PACKED(pack);
786
                } else {
787
                        unreachable("i16 not yet implemented");
788
789
790
                }
        }

791
        unreachable("Unknown convert");
792
793
}

Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
794
795
796
797
798
799
800
801
static unsigned
bi_pack_fma(bi_clause *clause, bi_bundle bundle, struct bi_registers *regs)
{
        if (!bundle.fma)
                return BIFROST_FMA_NOP;

        switch (bundle.fma->type) {
        case BI_ADD:
802
                return bi_pack_fma_addmin(bundle.fma, regs);
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
803
804
805
        case BI_CMP:
        case BI_BITWISE:
		return BIFROST_FMA_NOP;
806
807
        case BI_CONVERT:
		return bi_pack_fma_convert(bundle.fma, regs);
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
808
809
        case BI_CSEL:
		return bi_pack_fma_csel(bundle.fma, regs);
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
810
811
812
813
        case BI_FMA:
                return bi_pack_fma_fma(bundle.fma, regs);
        case BI_FREXP:
        case BI_ISUB:
814
                return BIFROST_FMA_NOP;
815
816
        case BI_MINMAX:
                return bi_pack_fma_addmin(bundle.fma, regs);
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
817
        case BI_MOV:
818
                return bi_pack_fma_1src(bundle.fma, regs, BIFROST_FMA_OP_MOV);
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
819
820
821
822
823
824
825
        case BI_SHIFT:
        case BI_SWIZZLE:
        case BI_ROUND:
		return BIFROST_FMA_NOP;
        default:
                unreachable("Cannot encode class as FMA");
        }
826
827
}

Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
828
static unsigned
829
bi_pack_add_ld_vary(bi_clause *clause, bi_instruction *ins, struct bi_registers *regs)
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
{
        unsigned size = nir_alu_type_get_type_size(ins->dest_type);
        assert(size == 32 || size == 16);

        unsigned op = (size == 32) ?
                BIFROST_ADD_OP_LD_VAR_32 :
                BIFROST_ADD_OP_LD_VAR_16;

        unsigned cmask = bi_from_bytemask(ins->writemask, size / 8);
        unsigned channels = util_bitcount(cmask);
        assert(cmask == ((1 << channels) - 1));

        unsigned packed_addr = 0;

        if (ins->src[0] & BIR_INDEX_CONSTANT) {
                /* Direct uses address field directly */
846
                packed_addr = bi_get_immediate(ins, ins->src[0]);
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
847
848
849
850
851
852
                assert(packed_addr < 0b1000);
        } else {
                /* Indirect gets an extra source */
                packed_addr = bi_get_src(ins, regs, 0, false) | 0b11000;
        }

853
854
855
856
        /* The destination is thrown in the data register */
        assert(ins->dest & BIR_INDEX_REGISTER);
        clause->data_register = ins->dest & ~BIR_INDEX_REGISTER;

Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
        assert(channels >= 1 && channels <= 4);

        struct bifrost_ld_var pack = {
                .src0 = bi_get_src(ins, regs, 1, false),
                .addr = packed_addr,
                .channels = MALI_POSITIVE(channels),
                .interp_mode = ins->load_vary.interp_mode,
                .reuse = ins->load_vary.reuse,
                .flat = ins->load_vary.flat,
                .op = op
        };

        RETURN_PACKED(pack);
}

Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
static unsigned
bi_pack_add_2src(bi_instruction *ins, struct bi_registers *regs, unsigned op)
{
        struct bifrost_add_2src pack = {
                .src0 = bi_get_src(ins, regs, 0, true),
                .src1 = bi_get_src(ins, regs, 1, true),
                .op = op
        };

        RETURN_PACKED(pack);
}

static unsigned
bi_pack_add_ld_ubo(bi_clause *clause, bi_instruction *ins, struct bi_registers *regs)
{
        unsigned components = bi_load32_components(ins);

        const unsigned ops[4] = {
                BIFROST_ADD_OP_LD_UBO_1,
                BIFROST_ADD_OP_LD_UBO_2,
                BIFROST_ADD_OP_LD_UBO_3,
                BIFROST_ADD_OP_LD_UBO_4
        };

        bi_write_data_register(clause, ins);
        return bi_pack_add_2src(ins, regs, ops[components - 1]);
}

Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
static enum bifrost_ldst_type
bi_pack_ldst_type(nir_alu_type T)
{
        switch (T) {
        case nir_type_float16: return BIFROST_LDST_F16;
        case nir_type_float32: return BIFROST_LDST_F32;
        case nir_type_int32:   return BIFROST_LDST_I32;
        case nir_type_uint32:  return BIFROST_LDST_U32;
        default: unreachable("Invalid type loaded");
        }
}

static unsigned
bi_pack_add_ld_var_addr(bi_clause *clause, bi_instruction *ins, struct bi_registers *regs)
{
        struct bifrost_ld_var_addr pack = {
                .src0 = bi_get_src(ins, regs, 1, false),
                .src1 = bi_get_src(ins, regs, 2, false),
918
                .location = bi_get_immediate(ins, ins->src[0]),
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
919
920
921
922
923
924
925
926
                .type = bi_pack_ldst_type(ins->src_types[3]),
                .op = BIFROST_ADD_OP_LD_VAR_ADDR
        };

        bi_write_data_register(clause, ins);
        RETURN_PACKED(pack);
}

Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
927
928
929
930
931
932
static unsigned
bi_pack_add_ld_attr(bi_clause *clause, bi_instruction *ins, struct bi_registers *regs)
{
        struct bifrost_ld_attr pack = {
                .src0 = bi_get_src(ins, regs, 1, false),
                .src1 = bi_get_src(ins, regs, 2, false),
933
                .location = bi_get_immediate(ins, ins->src[0]),
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
934
935
936
937
938
939
940
941
942
                .channels = MALI_POSITIVE(bi_load32_components(ins)),
                .type = bi_pack_ldst_type(ins->dest_type),
                .op = BIFROST_ADD_OP_LD_ATTR
        };

        bi_write_data_register(clause, ins);
        RETURN_PACKED(pack);
}

Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
static unsigned
bi_pack_add_st_vary(bi_clause *clause, bi_instruction *ins, struct bi_registers *regs)
{
        assert(ins->store_channels >= 1 && ins->store_channels <= 4);

        struct bifrost_st_vary pack = {
                .src0 = bi_get_src(ins, regs, 1, false),
                .src1 = bi_get_src(ins, regs, 2, false),
                .src2 = bi_get_src(ins, regs, 3, false),
                .channels = MALI_POSITIVE(ins->store_channels),
                .op = BIFROST_ADD_OP_ST_VAR
        };

        bi_read_data_register(clause, ins);
        RETURN_PACKED(pack);
}

Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
static unsigned
bi_pack_add_atest(bi_clause *clause, bi_instruction *ins, struct bi_registers *regs)
{
        /* TODO: fp16 */
        assert(ins->src_types[1] == nir_type_float32);

        struct bifrost_add_atest pack = {
                .src0 = bi_get_src(ins, regs, 0, false),
                .src1 = bi_get_src(ins, regs, 1, false),
                .component = 1, /* Set for fp32 */
                .op = BIFROST_ADD_OP_ATEST,
        };

        /* Despite *also* writing with the usual mechanism... quirky and
         * perhaps unnecessary, but let's match the blob */
        clause->data_register = ins->dest & ~BIR_INDEX_REGISTER;

        RETURN_PACKED(pack);
}

Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
980
981
982
983
984
985
986
987
988
989
990
991
992
993
static unsigned
bi_pack_add_blend(bi_instruction *ins, struct bi_registers *regs)
{
        struct bifrost_add_inst pack = {
                .src0 = bi_get_src(ins, regs, 0, false),
                .op = BIFROST_ADD_OP_BLEND
        };

        /* TODO: Pack location in uniform_const */
        assert(ins->blend_location == 0);

        RETURN_PACKED(pack);
}

994
995
996
997
static unsigned
bi_pack_add_special(bi_instruction *ins, struct bi_registers *regs)
{
        unsigned op = 0;
998
999
        bool fp16 = ins->dest_type == nir_type_float16;
        bool Y = ins->swizzle[0][0];
1000
1001

        if (ins->op.special == BI_SPECIAL_FRCP) {
1002
1003
1004
1005
                op = fp16 ?
                        (Y ? BIFROST_ADD_OP_FRCP_FAST_F16_Y :
                        BIFROST_ADD_OP_FRCP_FAST_F16_X) :
                        BIFROST_ADD_OP_FRCP_FAST_F32;
1006
        } else {
1007
1008
1009
1010
1011
                op = fp16 ?
                        (Y ? BIFROST_ADD_OP_FRSQ_FAST_F16_Y :
                        BIFROST_ADD_OP_FRSQ_FAST_F16_X) :
                        BIFROST_ADD_OP_FRSQ_FAST_F32;

1012
1013
1014
1015
1016
        }

        return bi_pack_add_1src(ins, regs, op);
}

1017
static unsigned
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
1018
bi_pack_add(bi_clause *clause, bi_bundle bundle, struct bi_registers *regs)
1019
{
1020
1021
1022
1023
1024
        if (!bundle.add)
                return BIFROST_ADD_NOP;

        switch (bundle.add->type) {
        case BI_ADD:
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
1025
                return BIFROST_ADD_NOP;
1026
        case BI_ATEST:
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
1027
                return bi_pack_add_atest(clause, bundle.add, regs);
1028
1029
        case BI_BRANCH:
        case BI_CMP:
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
1030
                return BIFROST_ADD_NOP;
1031
        case BI_BLEND:
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
1032
                return bi_pack_add_blend(bundle.add, regs);
1033
1034
1035
1036
1037
1038
        case BI_BITWISE:
        case BI_CONVERT:
        case BI_DISCARD:
        case BI_FREXP:
        case BI_ISUB:
        case BI_LOAD:
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
1039
                return BIFROST_ADD_NOP;
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
1040
1041
        case BI_LOAD_ATTR:
                return bi_pack_add_ld_attr(clause, bundle.add, regs);
1042
        case BI_LOAD_UNIFORM:
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
1043
                return bi_pack_add_ld_ubo(clause, bundle.add, regs);
1044
        case BI_LOAD_VAR:
1045
                return bi_pack_add_ld_vary(clause, bundle.add, regs);
1046
        case BI_LOAD_VAR_ADDRESS:
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
1047
                return bi_pack_add_ld_var_addr(clause, bundle.add, regs);
1048
1049
1050
1051
        case BI_MINMAX:
        case BI_MOV:
        case BI_SHIFT:
        case BI_STORE:
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
1052
                return BIFROST_ADD_NOP;
1053
        case BI_STORE_VAR:
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
1054
                return bi_pack_add_st_vary(clause, bundle.add, regs);
1055
        case BI_SPECIAL:
1056
                return bi_pack_add_special(bundle.add, regs);
1057
1058
1059
1060
1061
1062
1063
        case BI_SWIZZLE:
        case BI_TEX:
        case BI_ROUND:
                return BIFROST_ADD_NOP;
        default:
                unreachable("Cannot encode class as ADD");
        }
1064
1065
1066
1067
1068
1069
1070
1071
}

struct bi_packed_bundle {
        uint64_t lo;
        uint64_t hi;
};

static struct bi_packed_bundle
1072
bi_pack_bundle(bi_clause *clause, bi_bundle bundle, bi_bundle prev, bool first_bundle)
1073
{
1074
        struct bi_registers regs = bi_assign_ports(bundle, prev);
1075
        bi_assign_uniform_constant(clause, &regs, bundle);
1076
        regs.first_instruction = first_bundle;
1077
1078

        uint64_t reg = bi_pack_registers(regs);
Alyssa Rosenzweig's avatar
Alyssa Rosenzweig committed
1079
1080
        uint64_t fma = bi_pack_fma(clause, bundle, &regs);
        uint64_t add = bi_pack_add(clause, bundle, &regs);
1081
1082
1083
1084
1085
1086
1087
1088
1089

        struct bi_packed_bundle packed = {
                .lo = reg | (fma << 35) | ((add & 0b111111) << 58),
                .hi = add >> 6
        };

        return packed;
}

1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
/* Packs the next two constants as a dedicated constant quadword at the end of
 * the clause, returning the number packed. */

static unsigned
bi_pack_constants(bi_context *ctx, bi_clause *clause,
                unsigned index,
                struct util_dynarray *emission)
{
        /* After these two, are we done? Determines tag */
        bool done = clause->constant_count <= (index + 2);
        bool only = clause->constant_count <= (index + 1);

        /* TODO: Pos */
        assert(index == 0 && clause->bundle_count == 1);

        struct bifrost_fmt_constant quad = {
                .pos = 0, /* TODO */
                .tag = done ? BIFROST_FMTC_FINAL : BIFROST_FMTC_CONSTANTS,
                .imm_1 = clause->constants[index + 0] >> 4,
                .imm_2 = only ? 0 : clause->constants[index + 1] >> 4
        };

1112
1113
1114
1115
1116
        /* XXX: On G71, Connor observed that the difference of the top 4 bits
         * of the second constant with the first must be less than 8, otherwise
         * we have to swap them. I am not able to reproduce this on G52,
         * further investigation needed. Possibly an errata. XXX */

1117
1118
1119
1120
1121
        util_dynarray_append(emission, struct bifrost_fmt_constant, quad);

        return 2;
}

1122
1123
1124
1125
static void
bi_pack_clause(bi_context *ctx, bi_clause *clause, bi_clause *next,
                struct util_dynarray *emission)
{
1126
        struct bi_packed_bundle ins_1 = bi_pack_bundle(clause, clause->bundles[0], clause->bundles[0], true);
1127
1128
        assert(clause->bundle_count == 1);

1129
1130
1131
        /* Used to decide if we elide writes */
        bool is_fragment = ctx->stage == MESA_SHADER_FRAGMENT;

1132
1133
1134
        /* State for packing constants throughout */
        unsigned constant_index = 0;

1135
        struct bifrost_fmt1 quad_1 = {
1136
                .tag = clause->constant_count ? BIFROST_FMT1_CONSTANTS : BIFROST_FMT1_FINAL,
1137
                .header = bi_pack_header(clause, next, is_fragment),
1138
1139
1140
                .ins_1 = ins_1.lo,
                .ins_2 = ins_1.hi & ((1 << 11) - 1),
                .ins_0 = (ins_1.hi >> 11) & 0b111,
1141
1142
1143
        };

        util_dynarray_append(emission, struct bifrost_fmt1, quad_1);
1144
1145
1146
1147
1148
1149
1150

        /* Pack the remaining constants */

        while (constant_index < clause->constant_count) {
                constant_index += bi_pack_constants(ctx, clause,
                                constant_index, emission);
        }
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
}

static bi_clause *
bi_next_clause(bi_context *ctx, pan_block *block, bi_clause *clause)
{
        /* Try the next clause in this block */
        if (clause->link.next != &((bi_block *) block)->clauses)
                return list_first_entry(&(clause->link), bi_clause, link);

        /* Try the next block, or the one after that if it's empty, etc .*/
        pan_block *next_block = pan_next_block(block);

        bi_foreach_block_from(ctx, next_block, block) {
                bi_block *blk = (bi_block *) block;

                if (!list_is_empty(&blk->clauses))
                        return list_first_entry(&(blk->clauses), bi_clause, link);
        }

        return NULL;
}

void
bi_pack(bi_context *ctx, struct util_dynarray *emission)
{
        util_dynarray_init(emission, NULL);

        bi_foreach_block(ctx, _block) {
                bi_block *block = (bi_block *) _block;

                bi_foreach_clause_in_block(block, clause) {
                        bi_clause *next = bi_next_clause(ctx, _block, clause);
                        bi_pack_clause(ctx, clause, next, emission);
                }
        }
}