nvc0_vbo_translate.c 23.2 KB
Newer Older
1 2 3 4

#include "pipe/p_context.h"
#include "pipe/p_state.h"
#include "util/u_inlines.h"
5
#include "util/format/u_format.h"
6 7
#include "translate/translate.h"

8 9
#include "nvc0/nvc0_context.h"
#include "nvc0/nvc0_resource.h"
10

11
#include "nvc0/nvc0_3d.xml.h"
12 13 14 15 16 17 18 19 20 21

struct push_context {
   struct nouveau_pushbuf *push;

   struct translate *translate;
   void *dest;
   const void *idxbuf;

   uint32_t vertex_size;
   uint32_t restart_index;
22
   uint32_t start_instance;
23 24
   uint32_t instance_id;

25 26
   bool prim_restart;
   bool need_vertex_id;
27 28

   struct {
29 30
      bool enabled;
      bool value;
31
      uint8_t width;
32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
      unsigned stride;
      const uint8_t *data;
   } edgeflag;
};

static void nvc0_push_upload_vertex_ids(struct push_context *,
                                        struct nvc0_context *,
                                        const struct pipe_draw_info *);

static void
nvc0_push_context_init(struct nvc0_context *nvc0, struct push_context *ctx)
{
   ctx->push = nvc0->base.pushbuf;

   ctx->translate = nvc0->vertex->translate;
   ctx->vertex_size = nvc0->vertex->size;
48
   ctx->instance_id = 0;
49 50 51 52

   ctx->need_vertex_id =
      nvc0->vertprog->vp.need_vertex_id && (nvc0->vertex->num_elements < 32);

53
   ctx->edgeflag.value = true;
54 55 56 57 58
   ctx->edgeflag.enabled = nvc0->vertprog->vp.edgeflag < PIPE_MAX_ATTRIBS;

   /* silence warnings */
   ctx->edgeflag.data = NULL;
   ctx->edgeflag.stride = 0;
59
   ctx->edgeflag.width = 0;
60 61
}

62
static inline void
63 64 65 66 67 68 69 70 71
nvc0_vertex_configure_translate(struct nvc0_context *nvc0, int32_t index_bias)
{
   struct translate *translate = nvc0->vertex->translate;
   unsigned i;

   for (i = 0; i < nvc0->num_vtxbufs; ++i) {
      const uint8_t *map;
      const struct pipe_vertex_buffer *vb = &nvc0->vtxbuf[i];

72 73
      if (likely(vb->is_user_buffer))
         map = (const uint8_t *)vb->buffer.user;
74 75 76 77
      else {
         if (!vb->buffer.resource)
            continue;

78
         map = nouveau_resource_map_offset(&nvc0->base,
79
            nv04_resource(vb->buffer.resource), vb->buffer_offset, NOUVEAU_BO_RD);
80
      }
81 82 83 84 85 86 87 88

      if (index_bias && !unlikely(nvc0->vertex->instance_bufs & (1 << i)))
         map += (intptr_t)index_bias * vb->stride;

      translate->set_buffer(translate, i, map, vb->stride, ~0);
   }
}

89
static inline void
90
nvc0_push_map_idxbuf(struct push_context *ctx, struct nvc0_context *nvc0,
91
                     const struct pipe_draw_info *info)
92
{
93 94
   if (!info->has_user_indices) {
      struct nv04_resource *buf = nv04_resource(info->index.resource);
95 96
      ctx->idxbuf = nouveau_resource_map_offset(
            &nvc0->base, buf, 0, NOUVEAU_BO_RD);
97
   } else {
98
      ctx->idxbuf = info->index.user;
99
   }
100 101
}

102
static inline void
103 104 105 106 107 108
nvc0_push_map_edgeflag(struct push_context *ctx, struct nvc0_context *nvc0,
                       int32_t index_bias)
{
   unsigned attr = nvc0->vertprog->vp.edgeflag;
   struct pipe_vertex_element *ve = &nvc0->vertex->element[attr].pipe;
   struct pipe_vertex_buffer *vb = &nvc0->vtxbuf[ve->vertex_buffer_index];
109
   struct nv04_resource *buf = nv04_resource(vb->buffer.resource);
110 111

   ctx->edgeflag.stride = vb->stride;
112
   ctx->edgeflag.width = util_format_get_blocksize(ve->src_format);
113
   if (!vb->is_user_buffer) {
114 115
      unsigned offset = vb->buffer_offset + ve->src_offset;
      ctx->edgeflag.data = nouveau_resource_map_offset(&nvc0->base,
116
                           buf, offset, NOUVEAU_BO_RD);
117
   } else {
118
      ctx->edgeflag.data = (const uint8_t *)vb->buffer.user + ve->src_offset;
119 120
   }

121 122 123 124
   if (index_bias)
      ctx->edgeflag.data += (intptr_t)index_bias * vb->stride;
}

125
static inline unsigned
126 127 128 129 130 131 132
prim_restart_search_i08(const uint8_t *elts, unsigned push, uint8_t index)
{
   unsigned i;
   for (i = 0; i < push && elts[i] != index; ++i);
   return i;
}

133
static inline unsigned
134 135 136 137 138 139 140
prim_restart_search_i16(const uint16_t *elts, unsigned push, uint16_t index)
{
   unsigned i;
   for (i = 0; i < push && elts[i] != index; ++i);
   return i;
}

141
static inline unsigned
142 143 144 145 146 147 148
prim_restart_search_i32(const uint32_t *elts, unsigned push, uint32_t index)
{
   unsigned i;
   for (i = 0; i < push && elts[i] != index; ++i);
   return i;
}

149
static inline bool
150
ef_value_8(const struct push_context *ctx, uint32_t index)
151
{
152 153 154 155 156 157 158 159 160
   uint8_t *pf = (uint8_t *)&ctx->edgeflag.data[index * ctx->edgeflag.stride];
   return !!*pf;
}

static inline bool
ef_value_32(const struct push_context *ctx, uint32_t index)
{
   uint32_t *pf = (uint32_t *)&ctx->edgeflag.data[index * ctx->edgeflag.stride];
   return !!*pf;
161 162
}

163
static inline bool
164 165 166 167 168 169
ef_toggle(struct push_context *ctx)
{
   ctx->edgeflag.value = !ctx->edgeflag.value;
   return ctx->edgeflag.value;
}

170
static inline unsigned
171 172 173
ef_toggle_search_i08(struct push_context *ctx, const uint8_t *elts, unsigned n)
{
   unsigned i;
174 175 176 177 178
   bool ef = ctx->edgeflag.value;
   if (ctx->edgeflag.width == 1)
      for (i = 0; i < n && ef_value_8(ctx, elts[i]) == ef; ++i);
   else
      for (i = 0; i < n && ef_value_32(ctx, elts[i]) == ef; ++i);
179 180 181
   return i;
}

182
static inline unsigned
183 184 185
ef_toggle_search_i16(struct push_context *ctx, const uint16_t *elts, unsigned n)
{
   unsigned i;
186 187 188 189 190
   bool ef = ctx->edgeflag.value;
   if (ctx->edgeflag.width == 1)
      for (i = 0; i < n && ef_value_8(ctx, elts[i]) == ef; ++i);
   else
      for (i = 0; i < n && ef_value_32(ctx, elts[i]) == ef; ++i);
191 192 193
   return i;
}

194
static inline unsigned
195 196 197
ef_toggle_search_i32(struct push_context *ctx, const uint32_t *elts, unsigned n)
{
   unsigned i;
198 199 200 201 202
   bool ef = ctx->edgeflag.value;
   if (ctx->edgeflag.width == 1)
      for (i = 0; i < n && ef_value_8(ctx, elts[i]) == ef; ++i);
   else
      for (i = 0; i < n && ef_value_32(ctx, elts[i]) == ef; ++i);
203 204 205
   return i;
}

206
static inline unsigned
207 208 209
ef_toggle_search_seq(struct push_context *ctx, unsigned start, unsigned n)
{
   unsigned i;
210 211 212 213 214
   bool ef = ctx->edgeflag.value;
   if (ctx->edgeflag.width == 1)
      for (i = 0; i < n && ef_value_8(ctx, start++) == ef; ++i);
   else
      for (i = 0; i < n && ef_value_32(ctx, start++) == ef; ++i);
215 216 217
   return i;
}

218
static inline void *
219 220 221 222 223 224 225 226 227 228 229 230
nvc0_push_setup_vertex_array(struct nvc0_context *nvc0, const unsigned count)
{
   struct nouveau_pushbuf *push = nvc0->base.pushbuf;
   struct nouveau_bo *bo;
   uint64_t va;
   const unsigned size = count * nvc0->vertex->size;

   void *const dest = nouveau_scratch_get(&nvc0->base, size, &va, &bo);

   BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_START_HIGH(0)), 2);
   PUSH_DATAh(push, va);
   PUSH_DATA (push, va);
Ben Skeggs's avatar
Ben Skeggs committed
231 232 233 234 235

   if (nvc0->screen->eng3d->oclass < TU102_3D_CLASS)
      BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_LIMIT_HIGH(0)), 2);
   else
      BEGIN_NVC0(push, SUBC_3D(TU102_3D_VERTEX_ARRAY_LIMIT_HIGH(0)), 2);
236 237 238
   PUSH_DATAh(push, va + size - 1);
   PUSH_DATA (push, va + size - 1);

239
   BCTX_REFN_bo(nvc0->bufctx_3d, 3D_VTX_TMP, NOUVEAU_BO_GART | NOUVEAU_BO_RD,
240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259
                bo);
   nouveau_pushbuf_validate(push);

   return dest;
}

static void
disp_vertices_i08(struct push_context *ctx, unsigned start, unsigned count)
{
   struct nouveau_pushbuf *push = ctx->push;
   struct translate *translate = ctx->translate;
   const uint8_t *restrict elts = (uint8_t *)ctx->idxbuf + start;
   unsigned pos = 0;

   do {
      unsigned nR = count;

      if (unlikely(ctx->prim_restart))
         nR = prim_restart_search_i08(elts, nR, ctx->restart_index);

260 261
      translate->run_elts8(translate, elts, nR,
                           ctx->start_instance, ctx->instance_id, ctx->dest);
262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293
      count -= nR;
      ctx->dest += nR * ctx->vertex_size;

      while (nR) {
         unsigned nE = nR;

         if (unlikely(ctx->edgeflag.enabled))
            nE = ef_toggle_search_i08(ctx, elts, nR);

         PUSH_SPACE(push, 4);
         if (likely(nE >= 2)) {
            BEGIN_NVC0(push, NVC0_3D(VERTEX_BUFFER_FIRST), 2);
            PUSH_DATA (push, pos);
            PUSH_DATA (push, nE);
         } else
         if (nE) {
            if (pos <= 0xff) {
               IMMED_NVC0(push, NVC0_3D(VB_ELEMENT_U32), pos);
            } else {
               BEGIN_NVC0(push, NVC0_3D(VB_ELEMENT_U32), 1);
               PUSH_DATA (push, pos);
            }
         }
         if (unlikely(nE != nR))
            IMMED_NVC0(push, NVC0_3D(EDGEFLAG), ef_toggle(ctx));

         pos += nE;
         elts += nE;
         nR -= nE;
      }
      if (count) {
         BEGIN_NVC0(push, NVC0_3D(VB_ELEMENT_U32), 1);
294
         PUSH_DATA (push, 0xffffffff);
295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316
         ++elts;
         ctx->dest += ctx->vertex_size;
         ++pos;
         --count;
      }
   } while (count);
}

static void
disp_vertices_i16(struct push_context *ctx, unsigned start, unsigned count)
{
   struct nouveau_pushbuf *push = ctx->push;
   struct translate *translate = ctx->translate;
   const uint16_t *restrict elts = (uint16_t *)ctx->idxbuf + start;
   unsigned pos = 0;

   do {
      unsigned nR = count;

      if (unlikely(ctx->prim_restart))
         nR = prim_restart_search_i16(elts, nR, ctx->restart_index);

317 318
      translate->run_elts16(translate, elts, nR,
                            ctx->start_instance, ctx->instance_id, ctx->dest);
319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350
      count -= nR;
      ctx->dest += nR * ctx->vertex_size;

      while (nR) {
         unsigned nE = nR;

         if (unlikely(ctx->edgeflag.enabled))
            nE = ef_toggle_search_i16(ctx, elts, nR);

         PUSH_SPACE(push, 4);
         if (likely(nE >= 2)) {
            BEGIN_NVC0(push, NVC0_3D(VERTEX_BUFFER_FIRST), 2);
            PUSH_DATA (push, pos);
            PUSH_DATA (push, nE);
         } else
         if (nE) {
            if (pos <= 0xff) {
               IMMED_NVC0(push, NVC0_3D(VB_ELEMENT_U32), pos);
            } else {
               BEGIN_NVC0(push, NVC0_3D(VB_ELEMENT_U32), 1);
               PUSH_DATA (push, pos);
            }
         }
         if (unlikely(nE != nR))
            IMMED_NVC0(push, NVC0_3D(EDGEFLAG), ef_toggle(ctx));

         pos += nE;
         elts += nE;
         nR -= nE;
      }
      if (count) {
         BEGIN_NVC0(push, NVC0_3D(VB_ELEMENT_U32), 1);
351
         PUSH_DATA (push, 0xffffffff);
352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373
         ++elts;
         ctx->dest += ctx->vertex_size;
         ++pos;
         --count;
      }
   } while (count);
}

static void
disp_vertices_i32(struct push_context *ctx, unsigned start, unsigned count)
{
   struct nouveau_pushbuf *push = ctx->push;
   struct translate *translate = ctx->translate;
   const uint32_t *restrict elts = (uint32_t *)ctx->idxbuf + start;
   unsigned pos = 0;

   do {
      unsigned nR = count;

      if (unlikely(ctx->prim_restart))
         nR = prim_restart_search_i32(elts, nR, ctx->restart_index);

374 375
      translate->run_elts(translate, elts, nR,
                          ctx->start_instance, ctx->instance_id, ctx->dest);
376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407
      count -= nR;
      ctx->dest += nR * ctx->vertex_size;

      while (nR) {
         unsigned nE = nR;

         if (unlikely(ctx->edgeflag.enabled))
            nE = ef_toggle_search_i32(ctx, elts, nR);

         PUSH_SPACE(push, 4);
         if (likely(nE >= 2)) {
            BEGIN_NVC0(push, NVC0_3D(VERTEX_BUFFER_FIRST), 2);
            PUSH_DATA (push, pos);
            PUSH_DATA (push, nE);
         } else
         if (nE) {
            if (pos <= 0xff) {
               IMMED_NVC0(push, NVC0_3D(VB_ELEMENT_U32), pos);
            } else {
               BEGIN_NVC0(push, NVC0_3D(VB_ELEMENT_U32), 1);
               PUSH_DATA (push, pos);
            }
         }
         if (unlikely(nE != nR))
            IMMED_NVC0(push, NVC0_3D(EDGEFLAG), ef_toggle(ctx));

         pos += nE;
         elts += nE;
         nR -= nE;
      }
      if (count) {
         BEGIN_NVC0(push, NVC0_3D(VB_ELEMENT_U32), 1);
408
         PUSH_DATA (push, 0xffffffff);
409 410 411 412 413 414 415 416 417 418 419 420 421 422 423
         ++elts;
         ctx->dest += ctx->vertex_size;
         ++pos;
         --count;
      }
   } while (count);
}

static void
disp_vertices_seq(struct push_context *ctx, unsigned start, unsigned count)
{
   struct nouveau_pushbuf *push = ctx->push;
   struct translate *translate = ctx->translate;
   unsigned pos = 0;

424 425 426
   /* XXX: This will read the data corresponding to the primitive restart index,
    *  maybe we should avoid that ?
    */
427 428
   translate->run(translate, start, count,
                  ctx->start_instance, ctx->instance_id, ctx->dest);
429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452
   do {
      unsigned nr = count;

      if (unlikely(ctx->edgeflag.enabled))
         nr = ef_toggle_search_seq(ctx, start + pos, nr);

      PUSH_SPACE(push, 4);
      if (likely(nr)) {
         BEGIN_NVC0(push, NVC0_3D(VERTEX_BUFFER_FIRST), 2);
         PUSH_DATA (push, pos);
         PUSH_DATA (push, nr);
      }
      if (unlikely(nr != count))
         IMMED_NVC0(push, NVC0_3D(EDGEFLAG), ef_toggle(ctx));

      pos += nr;
      count -= nr;
   } while (count);
}


#define NVC0_PRIM_GL_CASE(n) \
   case PIPE_PRIM_##n: return NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_##n

453
static inline unsigned
454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470
nvc0_prim_gl(unsigned prim)
{
   switch (prim) {
   NVC0_PRIM_GL_CASE(POINTS);
   NVC0_PRIM_GL_CASE(LINES);
   NVC0_PRIM_GL_CASE(LINE_LOOP);
   NVC0_PRIM_GL_CASE(LINE_STRIP);
   NVC0_PRIM_GL_CASE(TRIANGLES);
   NVC0_PRIM_GL_CASE(TRIANGLE_STRIP);
   NVC0_PRIM_GL_CASE(TRIANGLE_FAN);
   NVC0_PRIM_GL_CASE(QUADS);
   NVC0_PRIM_GL_CASE(QUAD_STRIP);
   NVC0_PRIM_GL_CASE(POLYGON);
   NVC0_PRIM_GL_CASE(LINES_ADJACENCY);
   NVC0_PRIM_GL_CASE(LINE_STRIP_ADJACENCY);
   NVC0_PRIM_GL_CASE(TRIANGLES_ADJACENCY);
   NVC0_PRIM_GL_CASE(TRIANGLE_STRIP_ADJACENCY);
Ilia Mirkin's avatar
Ilia Mirkin committed
471
   NVC0_PRIM_GL_CASE(PATCHES);
472 473 474 475 476
   default:
      return NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_POINTS;
   }
}

477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553
typedef struct {
   uint32_t count;
   uint32_t primCount;
   uint32_t first;
   uint32_t baseInstance;
} DrawArraysIndirectCommand;

typedef struct {
   uint32_t count;
   uint32_t primCount;
   uint32_t firstIndex;
   int32_t  baseVertex;
   uint32_t baseInstance;
} DrawElementsIndirectCommand;

void
nvc0_push_vbo_indirect(struct nvc0_context *nvc0, const struct pipe_draw_info *info)
{
   /* The strategy here is to just read the commands from the indirect buffer
    * and do the draws. This is suboptimal, but will only happen in the case
    * that conversion is required for FIXED or DOUBLE inputs.
    */
   struct nvc0_screen *screen = nvc0->screen;
   struct nouveau_pushbuf *push = nvc0->base.pushbuf;
   struct nv04_resource *buf = nv04_resource(info->indirect->buffer);
   struct nv04_resource *buf_count = nv04_resource(info->indirect->indirect_draw_count);
   unsigned i;

   unsigned draw_count = info->indirect->draw_count;
   if (buf_count) {
      uint32_t *count = nouveau_resource_map_offset(
            &nvc0->base, buf_count, info->indirect->indirect_draw_count_offset,
            NOUVEAU_BO_RD);
      draw_count = *count;
   }

   uint8_t *buf_data = nouveau_resource_map_offset(
            &nvc0->base, buf, info->indirect->offset, NOUVEAU_BO_RD);
   struct pipe_draw_info single = *info;
   single.indirect = NULL;
   for (i = 0; i < draw_count; i++, buf_data += info->indirect->stride) {
      if (info->index_size) {
         DrawElementsIndirectCommand *cmd = (void *)buf_data;
         single.start = info->start + cmd->firstIndex;
         single.count = cmd->count;
         single.start_instance = cmd->baseInstance;
         single.instance_count = cmd->primCount;
         single.index_bias = cmd->baseVertex;
      } else {
         DrawArraysIndirectCommand *cmd = (void *)buf_data;
         single.start = cmd->first;
         single.count = cmd->count;
         single.start_instance = cmd->baseInstance;
         single.instance_count = cmd->primCount;
      }

      if (nvc0->vertprog->vp.need_draw_parameters) {
         PUSH_SPACE(push, 9);
         BEGIN_NVC0(push, NVC0_3D(CB_SIZE), 3);
         PUSH_DATA (push, NVC0_CB_AUX_SIZE);
         PUSH_DATAh(push, screen->uniform_bo->offset + NVC0_CB_AUX_INFO(0));
         PUSH_DATA (push, screen->uniform_bo->offset + NVC0_CB_AUX_INFO(0));
         BEGIN_1IC0(push, NVC0_3D(CB_POS), 1 + 3);
         PUSH_DATA (push, NVC0_CB_AUX_DRAW_INFO);
         PUSH_DATA (push, single.index_bias);
         PUSH_DATA (push, single.start_instance);
         PUSH_DATA (push, single.drawid + i);
      }

      nvc0_push_vbo(nvc0, &single);
   }

   nouveau_resource_unmap(buf);
   if (buf_count)
      nouveau_resource_unmap(buf_count);
}

554 555 556 557 558 559 560 561 562 563 564 565 566
void
nvc0_push_vbo(struct nvc0_context *nvc0, const struct pipe_draw_info *info)
{
   struct push_context ctx;
   unsigned i, index_size;
   unsigned inst_count = info->instance_count;
   unsigned vert_count = info->count;
   unsigned prim;

   nvc0_push_context_init(nvc0, &ctx);

   nvc0_vertex_configure_translate(nvc0, info->index_bias);

567 568 569 570 571 572
   if (nvc0->state.index_bias) {
      /* this is already taken care of by translate */
      IMMED_NVC0(ctx.push, NVC0_3D(VB_ELEMENT_BASE), 0);
      nvc0->state.index_bias = 0;
   }

573 574 575
   if (unlikely(ctx.edgeflag.enabled))
      nvc0_push_map_edgeflag(&ctx, nvc0, info->index_bias);

576 577 578
   ctx.prim_restart = info->primitive_restart;
   ctx.restart_index = info->restart_index;

579 580 581 582 583 584 585 586 587
   if (info->primitive_restart) {
      /* NOTE: I hope we won't ever need that last index (~0).
       * If we do, we have to disable primitive restart here always and
       * use END,BEGIN to restart. (XXX: would that affect PrimitiveID ?)
       * We could also deactive PRIM_RESTART_WITH_DRAW_ARRAYS temporarily,
       * and add manual restart to disp_vertices_seq.
       */
      BEGIN_NVC0(ctx.push, NVC0_3D(PRIM_RESTART_ENABLE), 2);
      PUSH_DATA (ctx.push, 1);
588
      PUSH_DATA (ctx.push, info->index_size ? 0xffffffff : info->restart_index);
589 590 591 592 593 594
   } else
   if (nvc0->state.prim_restart) {
      IMMED_NVC0(ctx.push, NVC0_3D(PRIM_RESTART_ENABLE), 0);
   }
   nvc0->state.prim_restart = info->primitive_restart;

595
   if (info->index_size) {
596
      nvc0_push_map_idxbuf(&ctx, nvc0, info);
597
      index_size = info->index_size;
598
   } else {
599
      if (unlikely(info->indirect && info->indirect->count_from_stream_output)) {
600 601
         struct pipe_context *pipe = &nvc0->base.pipe;
         struct nvc0_so_target *targ;
602
         targ = nvc0_so_target(info->indirect->count_from_stream_output);
603
         pipe->get_query_result(pipe, targ->pq, true, (void *)&vert_count);
604 605 606 607 608 609
         vert_count /= targ->stride;
      }
      ctx.idxbuf = NULL; /* shut up warnings */
      index_size = 0;
   }

610
   ctx.start_instance = info->start_instance;
611 612 613 614 615 616 617 618 619 620 621 622

   prim = nvc0_prim_gl(info->mode);
   do {
      PUSH_SPACE(ctx.push, 9);

      ctx.dest = nvc0_push_setup_vertex_array(nvc0, vert_count);
      if (unlikely(!ctx.dest))
         break;

      if (unlikely(ctx.need_vertex_id))
         nvc0_push_upload_vertex_ids(&ctx, nvc0, info);

623 624
      if (nvc0->screen->eng3d->oclass < GM107_3D_CLASS)
         IMMED_NVC0(ctx.push, NVC0_3D(VERTEX_ARRAY_FLUSH), 0);
625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648
      BEGIN_NVC0(ctx.push, NVC0_3D(VERTEX_BEGIN_GL), 1);
      PUSH_DATA (ctx.push, prim);
      switch (index_size) {
      case 1:
         disp_vertices_i08(&ctx, info->start, vert_count);
         break;
      case 2:
         disp_vertices_i16(&ctx, info->start, vert_count);
         break;
      case 4:
         disp_vertices_i32(&ctx, info->start, vert_count);
         break;
      default:
         assert(index_size == 0);
         disp_vertices_seq(&ctx, info->start, vert_count);
         break;
      }
      PUSH_SPACE(ctx.push, 1);
      IMMED_NVC0(ctx.push, NVC0_3D(VERTEX_END_GL), 0);

      if (--inst_count) {
         prim |= NVC0_3D_VERTEX_BEGIN_GL_INSTANCE_NEXT;
         ++ctx.instance_id;
      }
649
      nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_3D_VTX_TMP);
650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671
      nouveau_scratch_done(&nvc0->base);
   } while (inst_count);


   /* reset state and unmap buffers (no-op) */

   if (unlikely(!ctx.edgeflag.value)) {
      PUSH_SPACE(ctx.push, 1);
      IMMED_NVC0(ctx.push, NVC0_3D(EDGEFLAG), 1);
   }

   if (unlikely(ctx.need_vertex_id)) {
      PUSH_SPACE(ctx.push, 4);
      IMMED_NVC0(ctx.push, NVC0_3D(VERTEX_ID_REPLACE), 0);
      BEGIN_NVC0(ctx.push, NVC0_3D(VERTEX_ATTRIB_FORMAT(1)), 1);
      PUSH_DATA (ctx.push,
                 NVC0_3D_VERTEX_ATTRIB_FORMAT_CONST |
                 NVC0_3D_VERTEX_ATTRIB_FORMAT_TYPE_FLOAT |
                 NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_32);
      IMMED_NVC0(ctx.push, NVC0_3D(VERTEX_ARRAY_FETCH(1)), 0);
   }

672 673
   if (info->index_size && !info->has_user_indices)
      nouveau_resource_unmap(nv04_resource(info->index.resource));
674
   for (i = 0; i < nvc0->num_vtxbufs; ++i)
675
      nouveau_resource_unmap(nv04_resource(nvc0->vtxbuf[i].buffer.resource));
676 677

   NOUVEAU_DRV_STAT(&nvc0->screen->base, draw_calls_fallback_count, 1);
678 679
}

680
static inline void
681 682 683 684 685 686 687
copy_indices_u8(uint32_t *dst, const uint8_t *elts, uint32_t bias, unsigned n)
{
   unsigned i;
   for (i = 0; i < n; ++i)
      dst[i] = elts[i] + bias;
}

688
static inline void
689 690 691 692 693 694 695
copy_indices_u16(uint32_t *dst, const uint16_t *elts, uint32_t bias, unsigned n)
{
   unsigned i;
   for (i = 0; i < n; ++i)
      dst[i] = elts[i] + bias;
}

696
static inline void
697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714
copy_indices_u32(uint32_t *dst, const uint32_t *elts, uint32_t bias, unsigned n)
{
   unsigned i;
   for (i = 0; i < n; ++i)
      dst[i] = elts[i] + bias;
}

static void
nvc0_push_upload_vertex_ids(struct push_context *ctx,
                            struct nvc0_context *nvc0,
                            const struct pipe_draw_info *info)

{
   struct nouveau_pushbuf *push = ctx->push;
   struct nouveau_bo *bo;
   uint64_t va;
   uint32_t *data;
   uint32_t format;
715
   unsigned index_size = info->index_size;
716 717 718 719 720 721 722 723
   unsigned i;
   unsigned a = nvc0->vertex->num_elements;

   if (!index_size || info->index_bias)
      index_size = 4;
   data = (uint32_t *)nouveau_scratch_get(&nvc0->base,
                                          info->count * index_size, &va, &bo);

724
   BCTX_REFN_bo(nvc0->bufctx_3d, 3D_VTX_TMP, NOUVEAU_BO_GART | NOUVEAU_BO_RD,
725 726 727
                bo);
   nouveau_pushbuf_validate(push);

728
   if (info->index_size) {
729 730 731
      if (!info->index_bias) {
         memcpy(data, ctx->idxbuf, info->count * index_size);
      } else {
732
         switch (info->index_size) {
733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777
         case 1:
            copy_indices_u8(data, ctx->idxbuf, info->index_bias, info->count);
            break;
         case 2:
            copy_indices_u16(data, ctx->idxbuf, info->index_bias, info->count);
            break;
         default:
            copy_indices_u32(data, ctx->idxbuf, info->index_bias, info->count);
            break;
         }
      }
   } else {
      for (i = 0; i < info->count; ++i)
         data[i] = i + (info->start + info->index_bias);
   }

   format = (1 << NVC0_3D_VERTEX_ATTRIB_FORMAT_BUFFER__SHIFT) |
      NVC0_3D_VERTEX_ATTRIB_FORMAT_TYPE_UINT;

   switch (index_size) {
   case 1:
      format |= NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_8;
      break;
   case 2:
      format |= NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_16;
      break;
   default:
      format |= NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_32;
      break;
   }

   PUSH_SPACE(push, 12);

   if (unlikely(nvc0->state.instance_elts & 2)) {
      nvc0->state.instance_elts &= ~2;
      IMMED_NVC0(push, NVC0_3D(VERTEX_ARRAY_PER_INSTANCE(1)), 0);
   }

   BEGIN_NVC0(push, NVC0_3D(VERTEX_ATTRIB_FORMAT(a)), 1);
   PUSH_DATA (push, format);

   BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_FETCH(1)), 3);
   PUSH_DATA (push, NVC0_3D_VERTEX_ARRAY_FETCH_ENABLE | index_size);
   PUSH_DATAh(push, va);
   PUSH_DATA (push, va);
Ben Skeggs's avatar
Ben Skeggs committed
778 779 780 781 782

   if (nvc0->screen->eng3d->oclass < TU102_3D_CLASS)
      BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_LIMIT_HIGH(1)), 2);
   else
      BEGIN_NVC0(push, SUBC_3D(TU102_3D_VERTEX_ARRAY_LIMIT_HIGH(1)), 2);
783 784 785 786 787 788 789 790 791
   PUSH_DATAh(push, va + info->count * index_size - 1);
   PUSH_DATA (push, va + info->count * index_size - 1);

#define NVC0_3D_VERTEX_ID_REPLACE_SOURCE_ATTR_X(a) \
   (((0x80 + (a) * 0x10) / 4) << NVC0_3D_VERTEX_ID_REPLACE_SOURCE__SHIFT)

   BEGIN_NVC0(push, NVC0_3D(VERTEX_ID_REPLACE), 1);
   PUSH_DATA (push, NVC0_3D_VERTEX_ID_REPLACE_SOURCE_ATTR_X(a) | 1);
}