gem_exec_schedule.c 35.1 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
/*
 * Copyright © 2016 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

24 25
#include "config.h"

26
#include <sys/poll.h>
27
#include <sys/ioctl.h>
28
#include <sched.h>
29
#include <signal.h>
30 31 32

#include "igt.h"
#include "igt_vgem.h"
33
#include "igt_rand.h"
34
#include "igt_sysfs.h"
35
#include "i915/gem_ring.h"
36 37 38 39 40

#define LO 0
#define HI 1
#define NOISE 2

41 42
#define MAX_PRIO LOCAL_I915_CONTEXT_MAX_USER_PRIORITY
#define MIN_PRIO LOCAL_I915_CONTEXT_MIN_USER_PRIORITY
43

44
#define MAX_ELSP_QLEN 16
45

46 47
#define MAX_ENGINES 16

48 49
#define MAX_CONTEXTS 1024

50 51
IGT_TEST_DESCRIPTION("Check that we can control the order of execution");

52 53 54 55 56
static inline
uint32_t __sync_read_u32(int fd, uint32_t handle, uint64_t offset)
{
	uint32_t value;

57 58
	gem_set_domain(fd, handle, /* No write hazard lies! */
		       I915_GEM_DOMAIN_GTT, I915_GEM_DOMAIN_GTT);
59 60 61 62 63 64 65 66
	gem_read(fd, handle, offset, &value, sizeof(value));

	return value;
}

static inline
void __sync_read_u32_count(int fd, uint32_t handle, uint32_t *dst, uint64_t size)
{
67 68
	gem_set_domain(fd, handle, /* No write hazard lies! */
		       I915_GEM_DOMAIN_GTT, I915_GEM_DOMAIN_GTT);
69 70 71
	gem_read(fd, handle, 0, dst, size);
}

72 73 74
static uint32_t __store_dword(int fd, uint32_t ctx, unsigned ring,
			      uint32_t target, uint32_t offset, uint32_t value,
			      uint32_t cork, unsigned write_domain)
75 76 77 78 79 80 81 82 83
{
	const int gen = intel_gen(intel_get_drm_devid(fd));
	struct drm_i915_gem_exec_object2 obj[3];
	struct drm_i915_gem_relocation_entry reloc;
	struct drm_i915_gem_execbuffer2 execbuf;
	uint32_t batch[16];
	int i;

	memset(&execbuf, 0, sizeof(execbuf));
84
	execbuf.buffers_ptr = to_user_pointer(obj + !cork);
85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102
	execbuf.buffer_count = 2 + !!cork;
	execbuf.flags = ring;
	if (gen < 6)
		execbuf.flags |= I915_EXEC_SECURE;
	execbuf.rsvd1 = ctx;

	memset(obj, 0, sizeof(obj));
	obj[0].handle = cork;
	obj[1].handle = target;
	obj[2].handle = gem_create(fd, 4096);

	memset(&reloc, 0, sizeof(reloc));
	reloc.target_handle = obj[1].handle;
	reloc.presumed_offset = 0;
	reloc.offset = sizeof(uint32_t);
	reloc.delta = offset;
	reloc.read_domains = I915_GEM_DOMAIN_INSTRUCTION;
	reloc.write_domain = write_domain;
103
	obj[2].relocs_ptr = to_user_pointer(&reloc);
104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122
	obj[2].relocation_count = 1;

	i = 0;
	batch[i] = MI_STORE_DWORD_IMM | (gen < 6 ? 1 << 22 : 0);
	if (gen >= 8) {
		batch[++i] = offset;
		batch[++i] = 0;
	} else if (gen >= 4) {
		batch[++i] = 0;
		batch[++i] = offset;
		reloc.offset += sizeof(uint32_t);
	} else {
		batch[i]--;
		batch[++i] = offset;
	}
	batch[++i] = value;
	batch[++i] = MI_BATCH_BUFFER_END;
	gem_write(fd, obj[2].handle, 0, batch, sizeof(batch));
	gem_execbuf(fd, &execbuf);
123 124 125 126 127 128 129 130 131 132 133

	return obj[2].handle;
}

static void store_dword(int fd, uint32_t ctx, unsigned ring,
			uint32_t target, uint32_t offset, uint32_t value,
			uint32_t cork, unsigned write_domain)
{
	gem_close(fd, __store_dword(fd, ctx, ring,
				    target, offset, value,
				    cork, write_domain));
134 135
}

136 137 138 139 140 141 142 143 144
static uint32_t create_highest_priority(int fd)
{
	uint32_t ctx = gem_context_create(fd);

	/*
	 * If there is no priority support, all contexts will have equal
	 * priority (and therefore the max user priority), so no context
	 * can overtake us, and we effectively can form a plug.
	 */
145
	__gem_context_set_priority(fd, ctx, MAX_PRIO);
146 147 148 149

	return ctx;
}

150
static void unplug_show_queue(int fd, struct igt_cork *c, unsigned int engine)
151
{
152
	igt_spin_t *spin[MAX_ELSP_QLEN];
153 154

	for (int n = 0; n < ARRAY_SIZE(spin); n++) {
155 156 157 158 159 160
		const struct igt_spin_factory opts = {
			.ctx = create_highest_priority(fd),
			.engine = engine,
		};
		spin[n] = __igt_spin_batch_factory(fd, &opts);
		gem_context_destroy(fd, opts.ctx);
161 162
	}

163
	igt_cork_unplug(c); /* batches will now be queued on the engine */
164 165
	igt_debugfs_dump(fd, "i915_engine_info");

166 167 168
	for (int n = 0; n < ARRAY_SIZE(spin); n++)
		igt_spin_batch_free(fd, spin[n]);

169 170
}

171 172
static void fifo(int fd, unsigned ring)
{
173 174
	IGT_CORK_HANDLE(cork);
	uint32_t scratch, plug;
175
	uint32_t result;
176 177 178

	scratch = gem_create(fd, 4096);

179
	plug = igt_cork_plug(&cork, fd);
180 181

	/* Same priority, same timeline, final result will be the second eb */
182 183
	store_dword(fd, 0, ring, scratch, 0, 1, plug, 0);
	store_dword(fd, 0, ring, scratch, 0, 2, plug, 0);
184

185
	unplug_show_queue(fd, &cork, ring);
186
	gem_close(fd, plug);
187

188
	result =  __sync_read_u32(fd, scratch, 0);
189 190
	gem_close(fd, scratch);

191
	igt_assert_eq_u32(result, 2);
192 193
}

194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218
static void independent(int fd, unsigned int engine)
{
	IGT_CORK_HANDLE(cork);
	uint32_t scratch, plug, batch;
	igt_spin_t *spin = NULL;
	unsigned int other;
	uint32_t *ptr;

	igt_require(engine != 0);

	scratch = gem_create(fd, 4096);
	ptr = gem_mmap__gtt(fd, scratch, 4096, PROT_READ);
	igt_assert_eq(ptr[0], 0);

	plug = igt_cork_plug(&cork, fd);

	/* Check that we can submit to engine while all others are blocked */
	for_each_physical_engine(fd, other) {
		if (other == engine)
			continue;

		if (!gem_can_store_dword(fd, other))
			continue;

		if (spin == NULL) {
219
			spin = __igt_spin_batch_new(fd, .engine = other);
220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261
		} else {
			struct drm_i915_gem_exec_object2 obj = {
				.handle = spin->handle,
			};
			struct drm_i915_gem_execbuffer2 eb = {
				.buffer_count = 1,
				.buffers_ptr = to_user_pointer(&obj),
				.flags = other,
			};
			gem_execbuf(fd, &eb);
		}

		store_dword(fd, 0, other, scratch, 0, other, plug, 0);
	}
	igt_require(spin);

	/* Same priority, but different timeline (as different engine) */
	batch = __store_dword(fd, 0, engine, scratch, 0, engine, plug, 0);

	unplug_show_queue(fd, &cork, engine);
	gem_close(fd, plug);

	gem_sync(fd, batch);
	igt_assert(!gem_bo_busy(fd, batch));
	igt_assert(gem_bo_busy(fd, spin->handle));
	gem_close(fd, batch);

	/* Only the local engine should be free to complete. */
	igt_assert(gem_bo_busy(fd, scratch));
	igt_assert_eq(ptr[0], engine);

	igt_spin_batch_free(fd, spin);
	gem_quiescent_gpu(fd);

	/* And we expect the others to have overwritten us, order unspecified */
	igt_assert(!gem_bo_busy(fd, scratch));
	igt_assert_neq(ptr[0], engine);

	munmap(ptr, 4096);
	gem_close(fd, scratch);
}

262 263 264
static void smoketest(int fd, unsigned ring, unsigned timeout)
{
	const int ncpus = sysconf(_SC_NPROCESSORS_ONLN);
265
	unsigned engines[MAX_ENGINES];
266 267 268
	unsigned nengine;
	unsigned engine;
	uint32_t scratch;
269
	uint32_t result[2 * ncpus];
270 271

	nengine = 0;
272
	if (ring == ALL_ENGINES) {
273
		for_each_physical_engine(fd, engine)
274 275 276
			engines[nengine++] = engine;
	} else {
		engines[nengine++] = ring;
277 278 279 280 281 282 283 284 285 286 287 288 289 290 291
	}
	igt_require(nengine);

	scratch = gem_create(fd, 4096);
	igt_fork(child, ncpus) {
		unsigned long count = 0;
		uint32_t ctx;

		hars_petruska_f54_1_random_perturb(child);

		ctx = gem_context_create(fd);
		igt_until_timeout(timeout) {
			int prio;

			prio = hars_petruska_f54_1_random_unsafe_max(MAX_PRIO - MIN_PRIO) + MIN_PRIO;
292
			gem_context_set_priority(fd, ctx, prio);
293 294 295 296 297 298 299 300 301 302 303 304 305 306

			engine = engines[hars_petruska_f54_1_random_unsafe_max(nengine)];
			store_dword(fd, ctx, engine, scratch,
				    8*child + 0, ~child,
				    0, 0);
			for (unsigned int step = 0; step < 8; step++)
				store_dword(fd, ctx, engine, scratch,
					    8*child + 4, count++,
					    0, 0);
		}
		gem_context_destroy(fd, ctx);
	}
	igt_waitchildren();

307
	__sync_read_u32_count(fd, scratch, result, sizeof(result));
308 309 310
	gem_close(fd, scratch);

	for (unsigned n = 0; n < ncpus; n++) {
311
		igt_assert_eq_u32(result[2 * n], ~n);
312 313 314 315 316 317
		/*
		 * Note this count is approximate due to unconstrained
		 * ordering of the dword writes between engines.
		 *
		 * Take the result with a pinch of salt.
		 */
318
		igt_info("Child[%d] completed %u cycles\n",  n, result[(2 * n) + 1]);
319 320 321
	}
}

322 323 324
static void reorder(int fd, unsigned ring, unsigned flags)
#define EQUAL 1
{
325 326
	IGT_CORK_HANDLE(cork);
	uint32_t scratch, plug;
327
	uint32_t result;
328 329 330
	uint32_t ctx[2];

	ctx[LO] = gem_context_create(fd);
331
	gem_context_set_priority(fd, ctx[LO], MIN_PRIO);
332 333

	ctx[HI] = gem_context_create(fd);
334
	gem_context_set_priority(fd, ctx[HI], flags & EQUAL ? MIN_PRIO : 0);
335 336

	scratch = gem_create(fd, 4096);
337
	plug = igt_cork_plug(&cork, fd);
338 339 340 341

	/* We expect the high priority context to be executed first, and
	 * so the final result will be value from the low priority context.
	 */
342 343
	store_dword(fd, ctx[LO], ring, scratch, 0, ctx[LO], plug, 0);
	store_dword(fd, ctx[HI], ring, scratch, 0, ctx[HI], plug, 0);
344

345
	unplug_show_queue(fd, &cork, ring);
346
	gem_close(fd, plug);
347 348 349 350

	gem_context_destroy(fd, ctx[LO]);
	gem_context_destroy(fd, ctx[HI]);

351
	result =  __sync_read_u32(fd, scratch, 0);
352 353 354
	gem_close(fd, scratch);

	if (flags & EQUAL) /* equal priority, result will be fifo */
355
		igt_assert_eq_u32(result, ctx[HI]);
356
	else
357
		igt_assert_eq_u32(result, ctx[LO]);
358 359 360 361
}

static void promotion(int fd, unsigned ring)
{
362
	IGT_CORK_HANDLE(cork);
363
	uint32_t result, dep;
364
	uint32_t result_read, dep_read;
365
	uint32_t ctx[3];
366
	uint32_t plug;
367 368

	ctx[LO] = gem_context_create(fd);
369
	gem_context_set_priority(fd, ctx[LO], MIN_PRIO);
370 371

	ctx[HI] = gem_context_create(fd);
372
	gem_context_set_priority(fd, ctx[HI], 0);
373 374

	ctx[NOISE] = gem_context_create(fd);
375
	gem_context_set_priority(fd, ctx[NOISE], MIN_PRIO/2);
376 377 378 379

	result = gem_create(fd, 4096);
	dep = gem_create(fd, 4096);

380
	plug = igt_cork_plug(&cork, fd);
381 382 383 384 385 386

	/* Expect that HI promotes LO, so the order will be LO, HI, NOISE.
	 *
	 * fifo would be NOISE, LO, HI.
	 * strict priority would be  HI, NOISE, LO
	 */
387 388
	store_dword(fd, ctx[NOISE], ring, result, 0, ctx[NOISE], plug, 0);
	store_dword(fd, ctx[LO], ring, result, 0, ctx[LO], plug, 0);
389 390 391 392 393 394 395

	/* link LO <-> HI via a dependency on another buffer */
	store_dword(fd, ctx[LO], ring, dep, 0, ctx[LO], 0, I915_GEM_DOMAIN_INSTRUCTION);
	store_dword(fd, ctx[HI], ring, dep, 0, ctx[HI], 0, 0);

	store_dword(fd, ctx[HI], ring, result, 0, ctx[HI], 0, 0);

396
	unplug_show_queue(fd, &cork, ring);
397
	gem_close(fd, plug);
398 399 400 401 402

	gem_context_destroy(fd, ctx[NOISE]);
	gem_context_destroy(fd, ctx[LO]);
	gem_context_destroy(fd, ctx[HI]);

403
	dep_read = __sync_read_u32(fd, dep, 0);
404 405
	gem_close(fd, dep);

406
	result_read = __sync_read_u32(fd, result, 0);
407 408
	gem_close(fd, result);

409 410
	igt_assert_eq_u32(dep_read, ctx[HI]);
	igt_assert_eq_u32(result_read, ctx[NOISE]);
411 412
}

413 414
#define NEW_CTX (0x1 << 0)
#define HANG_LP (0x1 << 1)
415 416 417
static void preempt(int fd, unsigned ring, unsigned flags)
{
	uint32_t result = gem_create(fd, 4096);
418
	uint32_t result_read;
419
	igt_spin_t *spin[MAX_ELSP_QLEN];
420
	uint32_t ctx[2];
421
	igt_hang_t hang;
422 423

	ctx[LO] = gem_context_create(fd);
424
	gem_context_set_priority(fd, ctx[LO], MIN_PRIO);
425 426

	ctx[HI] = gem_context_create(fd);
427
	gem_context_set_priority(fd, ctx[HI], MAX_PRIO);
428

429
	if (flags & HANG_LP)
430
		hang = igt_hang_ctx(fd, ctx[LO], ring, 0);
431

432
	for (int n = 0; n < ARRAY_SIZE(spin); n++) {
433 434 435
		if (flags & NEW_CTX) {
			gem_context_destroy(fd, ctx[LO]);
			ctx[LO] = gem_context_create(fd);
436
			gem_context_set_priority(fd, ctx[LO], MIN_PRIO);
437
		}
438 439 440
		spin[n] = __igt_spin_batch_new(fd,
					       .ctx = ctx[LO],
					       .engine = ring);
441 442 443 444
		igt_debug("spin[%d].handle=%d\n", n, spin[n]->handle);

		store_dword(fd, ctx[HI], ring, result, 0, n + 1, 0, I915_GEM_DOMAIN_RENDER);

445 446
		result_read = __sync_read_u32(fd, result, 0);
		igt_assert_eq_u32(result_read, n + 1);
447 448 449
		igt_assert(gem_bo_busy(fd, spin[0]->handle));
	}

450
	for (int n = 0; n < ARRAY_SIZE(spin); n++)
451 452
		igt_spin_batch_free(fd, spin[n]);

453 454 455
	if (flags & HANG_LP)
		igt_post_hang_ring(fd, hang);

456 457 458 459 460 461
	gem_context_destroy(fd, ctx[LO]);
	gem_context_destroy(fd, ctx[HI]);

	gem_close(fd, result);
}

462 463 464 465 466 467 468 469 470 471 472
#define CHAIN 0x1
#define CONTEXTS 0x2

static igt_spin_t *__noise(int fd, uint32_t ctx, int prio, igt_spin_t *spin)
{
	unsigned other;

	gem_context_set_priority(fd, ctx, prio);

	for_each_physical_engine(fd, other) {
		if (spin == NULL) {
473 474 475
			spin = __igt_spin_batch_new(fd,
						    .ctx = ctx,
						    .engine = other);
476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496
		} else {
			struct drm_i915_gem_exec_object2 obj = {
				.handle = spin->handle,
			};
			struct drm_i915_gem_execbuffer2 eb = {
				.buffer_count = 1,
				.buffers_ptr = to_user_pointer(&obj),
				.rsvd1 = ctx,
				.flags = other,
			};
			gem_execbuf(fd, &eb);
		}
	}

	return spin;
}

static void __preempt_other(int fd,
			    uint32_t *ctx,
			    unsigned int target, unsigned int primary,
			    unsigned flags)
497 498
{
	uint32_t result = gem_create(fd, 4096);
499
	uint32_t result_read[4096 / sizeof(uint32_t)];
500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524
	unsigned int n, i, other;

	n = 0;
	store_dword(fd, ctx[LO], primary,
		    result, (n + 1)*sizeof(uint32_t), n + 1,
		    0, I915_GEM_DOMAIN_RENDER);
	n++;

	if (flags & CHAIN) {
		for_each_physical_engine(fd, other) {
			store_dword(fd, ctx[LO], other,
				    result, (n + 1)*sizeof(uint32_t), n + 1,
				    0, I915_GEM_DOMAIN_RENDER);
			n++;
		}
	}

	store_dword(fd, ctx[HI], target,
		    result, (n + 1)*sizeof(uint32_t), n + 1,
		    0, I915_GEM_DOMAIN_RENDER);

	igt_debugfs_dump(fd, "i915_engine_info");
	gem_set_domain(fd, result, I915_GEM_DOMAIN_GTT, 0);

	n++;
525 526

	__sync_read_u32_count(fd, result, result_read, sizeof(result_read));
527
	for (i = 0; i <= n; i++)
528
		igt_assert_eq_u32(result_read[i], i);
529 530 531 532 533 534 535 536

	gem_close(fd, result);
}

static void preempt_other(int fd, unsigned ring, unsigned int flags)
{
	unsigned int primary;
	igt_spin_t *spin = NULL;
537 538 539 540 541 542 543 544 545 546 547 548 549
	uint32_t ctx[3];

	/* On each engine, insert
	 * [NOISE] spinner,
	 * [LOW] write
	 *
	 * Then on our target engine do a [HIGH] write which should then
	 * prompt its dependent LOW writes in front of the spinner on
	 * each engine. The purpose of this test is to check that preemption
	 * can cross engines.
	 */

	ctx[LO] = gem_context_create(fd);
550
	gem_context_set_priority(fd, ctx[LO], MIN_PRIO);
551 552

	ctx[NOISE] = gem_context_create(fd);
553
	spin = __noise(fd, ctx[NOISE], 0, NULL);
554 555

	ctx[HI] = gem_context_create(fd);
556
	gem_context_set_priority(fd, ctx[HI], MAX_PRIO);
557

558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576
	for_each_physical_engine(fd, primary) {
		igt_debug("Primary engine: %s\n", e__->name);
		__preempt_other(fd, ctx, ring, primary, flags);

	}

	igt_assert(gem_bo_busy(fd, spin->handle));
	igt_spin_batch_free(fd, spin);

	gem_context_destroy(fd, ctx[LO]);
	gem_context_destroy(fd, ctx[NOISE]);
	gem_context_destroy(fd, ctx[HI]);
}

static void __preempt_queue(int fd,
			    unsigned target, unsigned primary,
			    unsigned depth, unsigned flags)
{
	uint32_t result = gem_create(fd, 4096);
577
	uint32_t result_read[4096 / sizeof(uint32_t)];
578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606
	igt_spin_t *above = NULL, *below = NULL;
	unsigned int other, n, i;
	int prio = MAX_PRIO;
	uint32_t ctx[3] = {
		gem_context_create(fd),
		gem_context_create(fd),
		gem_context_create(fd),
	};

	for (n = 0; n < depth; n++) {
		if (flags & CONTEXTS) {
			gem_context_destroy(fd, ctx[NOISE]);
			ctx[NOISE] = gem_context_create(fd);
		}
		above = __noise(fd, ctx[NOISE], prio--, above);
	}

	gem_context_set_priority(fd, ctx[HI], prio--);

	for (; n < MAX_ELSP_QLEN; n++) {
		if (flags & CONTEXTS) {
			gem_context_destroy(fd, ctx[NOISE]);
			ctx[NOISE] = gem_context_create(fd);
		}
		below = __noise(fd, ctx[NOISE], prio--, below);
	}

	gem_context_set_priority(fd, ctx[LO], prio--);

607
	n = 0;
608 609 610 611
	store_dword(fd, ctx[LO], primary,
		    result, (n + 1)*sizeof(uint32_t), n + 1,
		    0, I915_GEM_DOMAIN_RENDER);
	n++;
612

613 614 615 616 617 618 619
	if (flags & CHAIN) {
		for_each_physical_engine(fd, other) {
			store_dword(fd, ctx[LO], other,
				    result, (n + 1)*sizeof(uint32_t), n + 1,
				    0, I915_GEM_DOMAIN_RENDER);
			n++;
		}
620
	}
621 622

	store_dword(fd, ctx[HI], target,
623 624 625
		    result, (n + 1)*sizeof(uint32_t), n + 1,
		    0, I915_GEM_DOMAIN_RENDER);

626
	igt_debugfs_dump(fd, "i915_engine_info");
627

628 629 630
	if (above) {
		igt_assert(gem_bo_busy(fd, above->handle));
		igt_spin_batch_free(fd, above);
631 632
	}

633 634
	gem_set_domain(fd, result, I915_GEM_DOMAIN_GTT, 0);

635 636
	__sync_read_u32_count(fd, result, result_read, sizeof(result_read));

637 638
	n++;
	for (i = 0; i <= n; i++)
639
		igt_assert_eq_u32(result_read[i], i);
640

641 642 643 644 645
	if (below) {
		igt_assert(gem_bo_busy(fd, below->handle));
		igt_spin_batch_free(fd, below);
	}

646 647 648 649 650 651 652
	gem_context_destroy(fd, ctx[LO]);
	gem_context_destroy(fd, ctx[NOISE]);
	gem_context_destroy(fd, ctx[HI]);

	gem_close(fd, result);
}

653 654 655 656 657 658 659 660 661 662
static void preempt_queue(int fd, unsigned ring, unsigned int flags)
{
	unsigned other;

	for_each_physical_engine(fd, other) {
		for (unsigned depth = 0; depth <= MAX_ELSP_QLEN; depth++)
			__preempt_queue(fd, ring, other, depth, flags);
	}
}

663 664 665
static void preempt_self(int fd, unsigned ring)
{
	uint32_t result = gem_create(fd, 4096);
666
	uint32_t result_read[4096 / sizeof(uint32_t)];
667
	igt_spin_t *spin[MAX_ELSP_QLEN];
668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684
	unsigned int other;
	unsigned int n, i;
	uint32_t ctx[3];

	/* On each engine, insert
	 * [NOISE] spinner,
	 * [self/LOW] write
	 *
	 * Then on our target engine do a [self/HIGH] write which should then
	 * preempt its own lower priority task on any engine.
	 */

	ctx[NOISE] = gem_context_create(fd);

	ctx[HI] = gem_context_create(fd);

	n = 0;
685
	gem_context_set_priority(fd, ctx[HI], MIN_PRIO);
686
	for_each_physical_engine(fd, other) {
687 688 689
		spin[n] = __igt_spin_batch_new(fd,
					       .ctx = ctx[NOISE],
					       .engine = other);
690 691 692 693 694
		store_dword(fd, ctx[HI], other,
			    result, (n + 1)*sizeof(uint32_t), n + 1,
			    0, I915_GEM_DOMAIN_RENDER);
		n++;
	}
695
	gem_context_set_priority(fd, ctx[HI], MAX_PRIO);
696 697 698 699 700 701 702 703 704 705 706
	store_dword(fd, ctx[HI], ring,
		    result, (n + 1)*sizeof(uint32_t), n + 1,
		    0, I915_GEM_DOMAIN_RENDER);

	gem_set_domain(fd, result, I915_GEM_DOMAIN_GTT, 0);

	for (i = 0; i < n; i++) {
		igt_assert(gem_bo_busy(fd, spin[i]->handle));
		igt_spin_batch_free(fd, spin[i]);
	}

707 708
	__sync_read_u32_count(fd, result, result_read, sizeof(result_read));

709 710
	n++;
	for (i = 0; i <= n; i++)
711
		igt_assert_eq_u32(result_read[i], i);
712 713 714 715 716 717 718

	gem_context_destroy(fd, ctx[NOISE]);
	gem_context_destroy(fd, ctx[HI]);

	gem_close(fd, result);
}

719 720 721 722 723 724 725 726 727
static void preemptive_hang(int fd, unsigned ring)
{
	igt_spin_t *spin[MAX_ELSP_QLEN];
	igt_hang_t hang;
	uint32_t ctx[2];

	ctx[HI] = gem_context_create(fd);
	gem_context_set_priority(fd, ctx[HI], MAX_PRIO);

728
	for (int n = 0; n < ARRAY_SIZE(spin); n++) {
729 730 731
		ctx[LO] = gem_context_create(fd);
		gem_context_set_priority(fd, ctx[LO], MIN_PRIO);

732 733 734
		spin[n] = __igt_spin_batch_new(fd,
					       .ctx = ctx[LO],
					       .engine = ring);
735 736 737 738

		gem_context_destroy(fd, ctx[LO]);
	}

739
	hang = igt_hang_ctx(fd, ctx[HI], ring, 0);
740 741
	igt_post_hang_ring(fd, hang);

742
	for (int n = 0; n < ARRAY_SIZE(spin); n++) {
743 744 745 746 747 748 749 750 751 752 753
		/* Current behavior is to execute requests in order of submission.
		 * This is subject to change as the scheduler evolve. The test should
		 * be updated to reflect such changes.
		 */
		igt_assert(gem_bo_busy(fd, spin[n]->handle));
		igt_spin_batch_free(fd, spin[n]);
	}

	gem_context_destroy(fd, ctx[HI]);
}

754 755 756
static void deep(int fd, unsigned ring)
{
#define XS 8
757 758
	const unsigned int max_req = MAX_PRIO - MIN_PRIO;
	const unsigned size = ALIGN(4*max_req, 4096);
759
	struct timespec tv = {};
760
	IGT_CORK_HANDLE(cork);
761
	unsigned int nreq;
762
	uint32_t plug;
763
	uint32_t result, dep[XS];
764
	uint32_t read_buf[size / sizeof(uint32_t)];
765
	uint32_t expected = 0;
766
	uint32_t *ctx;
767 768
	int dep_nreq;
	int n;
769

770
	ctx = malloc(sizeof(*ctx) * MAX_CONTEXTS);
771
	for (n = 0; n < MAX_CONTEXTS; n++) {
772 773 774
		ctx[n] = gem_context_create(fd);
	}

775 776 777 778 779
	nreq = gem_measure_ring_inflight(fd, ring, 0) / (4 * XS) * MAX_CONTEXTS;
	if (nreq > max_req)
		nreq = max_req;
	igt_info("Using %d requests (prio range %d)\n", nreq, max_req);

780
	result = gem_create(fd, size);
781
	for (int m = 0; m < XS; m ++)
782
		dep[m] = gem_create(fd, size);
783

784 785 786 787 788 789 790
	/* Bind all surfaces and contexts before starting the timeout. */
	{
		struct drm_i915_gem_exec_object2 obj[XS + 2];
		struct drm_i915_gem_execbuffer2 execbuf;
		const uint32_t bbe = MI_BATCH_BUFFER_END;

		memset(obj, 0, sizeof(obj));
791
		for (n = 0; n < XS; n++)
792 793 794 795 796 797 798 799 800
			obj[n].handle = dep[n];
		obj[XS].handle = result;
		obj[XS+1].handle = gem_create(fd, 4096);
		gem_write(fd, obj[XS+1].handle, 0, &bbe, sizeof(bbe));

		memset(&execbuf, 0, sizeof(execbuf));
		execbuf.buffers_ptr = to_user_pointer(obj);
		execbuf.buffer_count = XS + 2;
		execbuf.flags = ring;
801
		for (n = 0; n < MAX_CONTEXTS; n++) {
802 803 804 805 806 807 808
			execbuf.rsvd1 = ctx[n];
			gem_execbuf(fd, &execbuf);
		}
		gem_close(fd, obj[XS+1].handle);
		gem_sync(fd, result);
	}

809
	plug = igt_cork_plug(&cork, fd);
810 811

	/* Create a deep dependency chain, with a few branches */
812 813 814 815 816 817 818 819 820 821 822 823
	for (n = 0; n < nreq && igt_seconds_elapsed(&tv) < 2; n++) {
		const int gen = intel_gen(intel_get_drm_devid(fd));
		struct drm_i915_gem_exec_object2 obj[3];
		struct drm_i915_gem_relocation_entry reloc;
		struct drm_i915_gem_execbuffer2 eb = {
			.buffers_ptr = to_user_pointer(obj),
			.buffer_count = 3,
			.flags = ring | (gen < 6 ? I915_EXEC_SECURE : 0),
			.rsvd1 = ctx[n % MAX_CONTEXTS],
		};
		uint32_t batch[16];
		int i;
824

825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861
		memset(obj, 0, sizeof(obj));
		obj[0].handle = plug;

		memset(&reloc, 0, sizeof(reloc));
		reloc.presumed_offset = 0;
		reloc.offset = sizeof(uint32_t);
		reloc.delta = sizeof(uint32_t) * n;
		reloc.read_domains = I915_GEM_DOMAIN_RENDER;
		reloc.write_domain = I915_GEM_DOMAIN_RENDER;
		obj[2].handle = gem_create(fd, 4096);
		obj[2].relocs_ptr = to_user_pointer(&reloc);
		obj[2].relocation_count = 1;

		i = 0;
		batch[i] = MI_STORE_DWORD_IMM | (gen < 6 ? 1 << 22 : 0);
		if (gen >= 8) {
			batch[++i] = reloc.delta;
			batch[++i] = 0;
		} else if (gen >= 4) {
			batch[++i] = 0;
			batch[++i] = reloc.delta;
			reloc.offset += sizeof(uint32_t);
		} else {
			batch[i]--;
			batch[++i] = reloc.delta;
		}
		batch[++i] = eb.rsvd1;
		batch[++i] = MI_BATCH_BUFFER_END;
		gem_write(fd, obj[2].handle, 0, batch, sizeof(batch));

		gem_context_set_priority(fd, eb.rsvd1, MAX_PRIO - nreq + n);
		for (int m = 0; m < XS; m++) {
			obj[1].handle = dep[m];
			reloc.target_handle = obj[1].handle;
			gem_execbuf(fd, &eb);
		}
		gem_close(fd, obj[2].handle);
862
	}
863 864 865
	igt_info("First deptree: %d requests [%.3fs]\n",
		 n * XS, 1e-9*igt_nsec_elapsed(&tv));
	dep_nreq = n;
866

867
	for (n = 0; n < nreq && igt_seconds_elapsed(&tv) < 4; n++) {
868 869
		uint32_t context = ctx[n % MAX_CONTEXTS];
		gem_context_set_priority(fd, context, MAX_PRIO - nreq + n);
870 871

		for (int m = 0; m < XS; m++) {
872 873
			store_dword(fd, context, ring, result, 4*n, context, dep[m], 0);
			store_dword(fd, context, ring, result, 4*m, context, 0, I915_GEM_DOMAIN_INSTRUCTION);
874
		}
875
		expected = context;
876
	}
877 878
	igt_info("Second deptree: %d requests [%.3fs]\n",
		 n * XS, 1e-9*igt_nsec_elapsed(&tv));
879

880
	unplug_show_queue(fd, &cork, ring);
881
	gem_close(fd, plug);
882
	igt_require(expected); /* too slow */
883

884
	for (n = 0; n < MAX_CONTEXTS; n++)
885 886 887
		gem_context_destroy(fd, ctx[n]);

	for (int m = 0; m < XS; m++) {
888
		__sync_read_u32_count(fd, dep[m], read_buf, sizeof(read_buf));
889 890
		gem_close(fd, dep[m]);

891
		for (n = 0; n < dep_nreq; n++)
892
			igt_assert_eq_u32(read_buf[n], ctx[n % MAX_CONTEXTS]);
893 894
	}

895
	__sync_read_u32_count(fd, result, read_buf, sizeof(read_buf));
896 897
	gem_close(fd, result);

898
	/* No reordering due to PI on all contexts because of the common dep */
899
	for (int m = 0; m < XS; m++)
900
		igt_assert_eq_u32(read_buf[m], expected);
901 902

	free(ctx);
903 904 905
#undef XS
}

906 907 908 909 910 911
static void alarm_handler(int sig)
{
}

static int __execbuf(int fd, struct drm_i915_gem_execbuffer2 *execbuf)
{
912 913 914 915
	int err = 0;
	if (ioctl(fd, DRM_IOCTL_I915_GEM_EXECBUFFER2, execbuf))
		err = -errno;
	return err;
916 917
}

918 919
static void wide(int fd, unsigned ring)
{
920
	struct timespec tv = {};
921
	unsigned int ring_size = gem_measure_ring_inflight(fd, ring, MEASURE_RING_NEW_CTX);
922

923 924
	IGT_CORK_HANDLE(cork);
	uint32_t plug;
925
	uint32_t result;
926
	uint32_t result_read[MAX_CONTEXTS];
927
	uint32_t *ctx;
928
	unsigned int count;
929

930 931
	ctx = malloc(sizeof(*ctx)*MAX_CONTEXTS);
	for (int n = 0; n < MAX_CONTEXTS; n++)
932 933
		ctx[n] = gem_context_create(fd);

934
	result = gem_create(fd, 4*MAX_CONTEXTS);
935

936
	plug = igt_cork_plug(&cork, fd);
937 938

	/* Lots of in-order requests, plugged and submitted simultaneously */
939 940 941
	for (count = 0;
	     igt_seconds_elapsed(&tv) < 5 && count < ring_size;
	     count++) {
942
		for (int n = 0; n < MAX_CONTEXTS; n++) {
943
			store_dword(fd, ctx[n], ring, result, 4*n, ctx[n], plug, I915_GEM_DOMAIN_INSTRUCTION);
944
		}
945
	}
946
	igt_info("Submitted %d requests over %d contexts in %.1fms\n",
947
		 count, MAX_CONTEXTS, igt_nsec_elapsed(&tv) * 1e-6);
948

949
	unplug_show_queue(fd, &cork, ring);
950
	gem_close(fd, plug);
951

952
	for (int n = 0; n < MAX_CONTEXTS; n++)
953 954
		gem_context_destroy(fd, ctx[n]);

955
	__sync_read_u32_count(fd, result, result_read, sizeof(result_read));
956
	for (int n = 0; n < MAX_CONTEXTS; n++)
957
		igt_assert_eq_u32(result_read[n], ctx[n]);
958 959 960

	gem_close(fd, result);
	free(ctx);
961 962 963 964 965 966 967 968
}

static void reorder_wide(int fd, unsigned ring)
{
	const int gen = intel_gen(intel_get_drm_devid(fd));
	struct drm_i915_gem_relocation_entry reloc;
	struct drm_i915_gem_exec_object2 obj[3];
	struct drm_i915_gem_execbuffer2 execbuf;
969
	struct timespec tv = {};
970 971 972
	unsigned int ring_size = gem_measure_ring_inflight(fd, ring, MEASURE_RING_NEW_CTX);
	IGT_CORK_HANDLE(cork);
	uint32_t result, target, plug;
973 974
	uint32_t result_read[1024];
	uint32_t *expected;
975 976 977

	result = gem_create(fd, 4096);
	target = gem_create(fd, 4096);
978
	plug = igt_cork_plug(&cork, fd);
979

980
	expected = gem_mmap__cpu(fd, target, 0, 4096, PROT_WRITE);
981 982 983
	gem_set_domain(fd, target, I915_GEM_DOMAIN_CPU, I915_GEM_DOMAIN_CPU);

	memset(obj, 0, sizeof(obj));
984
	obj[0].handle = plug;
985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000
	obj[1].handle = result;
	obj[2].relocs_ptr = to_user_pointer(&reloc);
	obj[2].relocation_count = 1;

	memset(&reloc, 0, sizeof(reloc));
	reloc.target_handle = result;
	reloc.read_domains = I915_GEM_DOMAIN_INSTRUCTION;
	reloc.write_domain = 0; /* lies */

	memset(&execbuf, 0, sizeof(execbuf));
	execbuf.buffers_ptr = to_user_pointer(obj);
	execbuf.buffer_count = 3;
	execbuf.flags = ring;
	if (gen < 6)
		execbuf.flags |= I915_EXEC_SECURE;

1001 1002 1003 1004
	for (int n = MIN_PRIO, x = 1;
	     igt_seconds_elapsed(&tv) < 5 && n <= MAX_PRIO;
	     n++, x++) {
		unsigned int sz = ALIGN(ring_size * 64, 4096);
1005 1006 1007
		uint32_t *batch;

		execbuf.rsvd1 = gem_context_create(fd);
1008
		gem_context_set_priority(fd, execbuf.rsvd1, n);
1009

1010 1011
		obj[2].handle = gem_create(fd, sz);
		batch = gem_mmap__gtt(fd, obj[2].handle, sz, PROT_WRITE);
1012 1013
		gem_set_domain(fd, obj[2].handle, I915_GEM_DOMAIN_GTT, I915_GEM_DOMAIN_GTT);

1014
		for (int m = 0; m < ring_size; m++) {
1015
			uint64_t addr;
1016
			int idx = hars_petruska_f54_1_random_unsafe_max(1024);
1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039
			int i;

			execbuf.batch_start_offset = m * 64;
			reloc.offset = execbuf.batch_start_offset + sizeof(uint32_t);
			reloc.delta = idx * sizeof(uint32_t);
			addr = reloc.presumed_offset + reloc.delta;

			i = execbuf.batch_start_offset / sizeof(uint32_t);
			batch[i] = MI_STORE_DWORD_IMM | (gen < 6 ? 1 << 22 : 0);
			if (gen >= 8) {
				batch[++i] = addr;
				batch[++i] = addr >> 32;
			} else if (gen >= 4) {
				batch[++i] = 0;
				batch[++i] = addr;
				reloc.offset += sizeof(uint32_t);
			} else {
				batch[i]--;
				batch[++i] = addr;
			}
			batch[++i] = x;
			batch[++i] = MI_BATCH_BUFFER_END;

1040 1041
			if (!expected[idx])
				expected[idx] =  x;
1042 1043 1044 1045

			gem_execbuf(fd, &execbuf);
		}

1046
		munmap(batch, sz);
1047 1048 1049 1050
		gem_close(fd, obj[2].handle);
		gem_context_destroy(fd, execbuf.rsvd1);
	}

1051
	unplug_show_queue(fd, &cork, ring);
1052
	gem_close(fd, plug);
1053

1054
	__sync_read_u32_count(fd, result, result_read, sizeof(result_read));
1055
	for (int n = 0; n < 1024; n++)
1056 1057
		igt_assert_eq_u32(result_read[n], expected[n]);

1058
	munmap(expected, 4096);
1059 1060 1061

	gem_close(fd, result);
	gem_close(fd, target);
1062 1063
}

1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084
static void bind_to_cpu(int cpu)
{
	const int ncpus = sysconf(_SC_NPROCESSORS_ONLN);
	struct sched_param rt = {.sched_priority = 99 };
	cpu_set_t allowed;

	igt_assert(sched_setscheduler(getpid(), SCHED_RR | SCHED_RESET_ON_FORK, &rt) == 0);

	CPU_ZERO(&allowed);
	CPU_SET(cpu % ncpus, &allowed);
	igt_assert(sched_setaffinity(getpid(), sizeof(cpu_set_t), &allowed) == 0);
}

static void test_pi_ringfull(int fd, unsigned int engine)
{
	const uint32_t bbe = MI_BATCH_BUFFER_END;
	struct sigaction sa = { .sa_handler = alarm_handler };
	struct drm_i915_gem_execbuffer2 execbuf;
	struct drm_i915_gem_exec_object2 obj[2];
	unsigned int last, count;
	struct itimerval itv;
1085
	IGT_CORK_HANDLE(c);
1086
	uint32_t vip;
1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100
	bool *result;

	result = mmap(NULL, 4096, PROT_WRITE, MAP_SHARED | MAP_ANON, -1, 0);
	igt_assert(result != MAP_FAILED);

	memset(&execbuf, 0, sizeof(execbuf));
	memset(&obj, 0, sizeof(obj));

	obj[1].handle = gem_create(fd, 4096);
	gem_write(fd, obj[1].handle, 0, &bbe, sizeof(bbe));

	execbuf.buffers_ptr = to_user_pointer(&obj[1]);
	execbuf.buffer_count = 1;
	execbuf.flags = engine;
1101 1102

	/* Warm up both (hi/lo) contexts */
1103
	execbuf.rsvd1 = gem_context_create(fd);
1104 1105 1106 1107
	gem_context_set_priority(fd, execbuf.rsvd1, MAX_PRIO);
	gem_execbuf(fd, &execbuf);
	gem_sync(fd, obj[1].handle);
	vip = execbuf.rsvd1;
1108

1109 1110
	execbuf.rsvd1 = gem_context_create(fd);
	gem_context_set_priority(fd, execbuf.rsvd1, MIN_PRIO);
1111 1112 1113 1114
	gem_execbuf(fd, &execbuf);
	gem_sync(fd, obj[1].handle);

	/* Fill the low-priority ring */
1115
	obj[0].handle = igt_cork_plug(&c, fd);
1116 1117 1118 1119 1120 1121

	execbuf.buffers_ptr = to_user_pointer(obj);
	execbuf.buffer_count = 2;

	sigaction(SIGALRM, &sa, NULL);
	itv.it_interval.tv_sec = 0;
1122
	itv.it_interval.tv_usec = 1000;
1123
	itv.it_value.tv_sec = 0;
1124
	itv.it_value.tv_usec = 10000;
1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151
	setitimer(ITIMER_REAL, &itv, NULL);

	last = -1;
	count = 0;
	do {
		if (__execbuf(fd, &execbuf) == 0) {
			count++;
			continue;
		}

		if (last == count)
			break;

		last = count;
	} while (1);
	igt_debug("Filled low-priority ring with %d batches\n", count);

	memset(&itv, 0, sizeof(itv));
	setitimer(ITIMER_REAL, &itv, NULL);

	execbuf.buffers_ptr = to_user_pointer(&obj[1]);
	execbuf.buffer_count = 1;

	/* both parent + child on the same cpu, only parent is RT */
	bind_to_cpu(0);

	igt_fork(child, 1) {
1152 1153 1154
		int err;

		result[0] = vip != execbuf.rsvd1;
1155

1156
		igt_debug("Waking parent\n");
1157 1158 1159 1160
		kill(getppid(), SIGALRM);
		sched_yield();
		result[1] = true;

1161
		sigaction(SIGALRM, &sa, NULL);
1162 1163 1164 1165 1166 1167 1168 1169
		itv.it_value.tv_sec = 0;
		itv.it_value.tv_usec = 10000;
		setitimer(ITIMER_REAL, &itv, NULL);

		/* Since we are the high priority task, we expect to be
		 * able to add ourselves to *our* ring without interruption.
		 */
		igt_debug("HP child executing\n");
1170
		execbuf.rsvd1 = vip;
1171 1172 1173 1174 1175 1176 1177
		err = __execbuf(fd, &execbuf);
		igt_debug("HP execbuf returned %d\n", err);

		memset(&itv, 0, sizeof(itv));
		setitimer(ITIMER_REAL, &itv, NULL);

		result[2] = err == 0;
1178 1179 1180 1181
	}

	/* Relinquish CPU just to allow child to create a context */
	sleep(1);
1182
	igt_assert_f(result[0], "HP context (child) not created\n");
1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194
	igt_assert_f(!result[1], "Child released too early!\n");

	/* Parent sleeps waiting for ringspace, releasing child */
	itv.it_value.tv_sec = 0;
	itv.it_value.tv_usec = 50000;
	setitimer(ITIMER_REAL, &itv, NULL);
	igt_debug("LP parent executing\n");
	igt_assert_eq(__execbuf(fd, &execbuf), -EINTR);
	igt_assert_f(result[1], "Child was not released!\n");
	igt_assert_f(result[2],
		     "High priority child unable to submit within 10ms\n");

1195
	igt_cork_unplug(&c);
1196 1197 1198
	igt_waitchildren();

	gem_context_destroy(fd, execbuf.rsvd1);
1199
	gem_context_destroy(fd, vip);
1200 1201 1202 1203 1204
	gem_close(fd, obj[1].handle);
	gem_close(fd, obj[0].handle);
	munmap(result, 4096);
}

1205 1206 1207 1208 1209 1210 1211 1212 1213
igt_main
{
	const struct intel_execution_engine *e;
	int fd = -1;

	igt_skip_on_simulation();

	igt_fixture {
		fd = drm_open_driver_master(DRIVER_INTEL);
1214
		gem_submission_print_method(fd);
1215
		gem_scheduler_print_capability(fd);
1216

1217
		igt_require_gem(fd);
1218
		gem_require_mmap_wc(fd);
1219 1220
		gem_require_contexts(fd);

1221 1222 1223 1224 1225 1226 1227 1228 1229 1230
		igt_fork_hang_detector(fd);
	}

	igt_subtest_group {
		for (e = intel_execution_engines; e->name; e++) {
			/* default exec-id is purely symbolic */
			if (e->exec_id == 0)
				continue;

			igt_subtest_f("fifo-%s", e->name) {
1231
				igt_require(gem_ring_has_physical_engine(fd, e->exec_id | e->flags));
1232
				igt_require(gem_can_store_dword(fd, e->exec_id | e->flags));
1233 1234
				fifo(fd, e->exec_id | e->flags);
			}
1235 1236 1237 1238 1239 1240

			igt_subtest_f("independent-%s", e->name) {
				igt_require(gem_ring_has_physical_engine(fd, e->exec_id | e->flags));
				igt_require(gem_can_store_dword(fd, e->exec_id | e->flags));
				independent(fd, e->exec_id | e->flags);
			}
1241 1242 1243 1244 1245
		}
	}

	igt_subtest_group {
		igt_fixture {
1246 1247
			igt_require(gem_scheduler_enabled(fd));
			igt_require(gem_scheduler_has_ctx_priority(fd));
1248 1249
		}

1250
		igt_subtest("smoketest-all")
1251
			smoketest(fd, ALL_ENGINES, 30);
1252

1253 1254 1255 1256 1257
		for (e = intel_execution_engines; e->name; e++) {
			if (e->exec_id == 0)
				continue;

			igt_subtest_group {
1258
				igt_fixture {
1259
					igt_require(gem_ring_has_physical_engine(fd, e->exec_id | e->flags));
1260
					igt_require(gem_can_store_dword(fd, e->exec_id | e->flags));
1261
				}
1262 1263 1264 1265 1266 1267 1268 1269 1270 1271

				igt_subtest_f("in-order-%s", e->name)
					reorder(fd, e->exec_id | e->flags, EQUAL);

				igt_subtest_f("out-order-%s", e->name)
					reorder(fd, e->exec_id | e->flags, 0);

				igt_subtest_f("promotion-%s", e->name)
					promotion(fd, e->exec_id | e->flags);

1272 1273
				igt_subtest_group {
					igt_fixture {
1274
						igt_require(gem_scheduler_has_preemption(fd));
1275
					}
1276

1277 1278
					igt_subtest_f("preempt-%s", e->name)
						preempt(fd, e->exec_id | e->flags, 0);
1279

1280 1281
					igt_subtest_f("preempt-contexts-%s", e->name)
						preempt(fd, e->exec_id | e->flags, NEW_CTX);
1282

1283 1284
					igt_subtest_f("preempt-self-%s", e->name)
						preempt_self(fd, e->exec_id | e->flags);
1285

1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302
					igt_subtest_f("preempt-other-%s", e->name)
						preempt_other(fd, e->exec_id | e->flags, 0);

					igt_subtest_f("preempt-other-chain-%s", e->name)
						preempt_other(fd, e->exec_id | e->flags, CHAIN);

					igt_subtest_f("preempt-queue-%s", e->name)
						preempt_queue(fd, e->exec_id | e->flags, 0);

					igt_subtest_f("preempt-queue-chain-%s", e->name)
						preempt_queue(fd, e->exec_id | e->flags, CHAIN);
					igt_subtest_f("preempt-queue-contexts-%s", e->name)
						preempt_queue(fd, e->exec_id | e->flags, CONTEXTS);

					igt_subtest_f("preempt-queue-contexts-chain-%s", e->name)
						preempt_queue(fd, e->exec_id | e->flags, CONTEXTS | CHAIN);

1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314
					igt_subtest_group {
						igt_hang_t hang;

						igt_fixture {
							igt_stop_hang_detector();
							hang = igt_allow_hang(fd, 0, 0);
						}

						igt_subtest_f("preempt-hang-%s", e->name) {
							preempt(fd, e->exec_id | e->flags, NEW_CTX | HANG_LP);
						}

1315 1316 1317
						igt_subtest_f("preemptive-hang-%s", e->name)
							preemptive_hang(fd, e->exec_id | e->flags);

1318 1319 1320 1321 1322
						igt_fixture {
							igt_disallow_hang(fd, hang);
							igt_fork_hang_detector(fd);
						}
					}
1323
				}
1324

1325 1326
				igt_subtest_f("deep-%s", e->name)
					deep(fd, e->exec_id | e->flags);
1327 1328 1329

				igt_subtest_f("wide-%s", e->name)
					wide(fd, e->exec_id | e->flags);
1330 1331 1332

				igt_subtest_f("reorder-wide-%s", e->name)
					reorder_wide(fd, e->exec_id | e->flags);
1333 1334 1335

				igt_subtest_f("smoketest-%s", e->name)
					smoketest(fd, e->exec_id | e->flags, 5);
1336 1337 1338 1339
			}
		}
	}

1340 1341
	igt_subtest_group {
		igt_fixture {
1342 1343
			igt_require(gem_scheduler_enabled(fd));
			igt_require(gem_scheduler_has_ctx_priority(fd));
1344 1345

			/* need separate rings */
1346
			igt_require(gem_has_execlists(fd));
1347 1348 1349
		}

		for (e = intel_execution_engines; e->name; e++) {
1350 1351 1352
			if (e->exec_id == 0)
				continue;

1353 1354
			igt_subtest_group {
				igt_fixture {
1355
					igt_require(gem_ring_has_physical_engine(fd, e->exec_id | e->flags));
1356
					igt_require(gem_scheduler_has_preemption(fd));
1357 1358 1359 1360 1361 1362 1363 1364
				}

				igt_subtest_f("pi-ringfull-%s", e->name)
					test_pi_ringfull(fd, e->exec_id | e->flags);
			}
		}
	}

1365 1366 1367 1368 1369
	igt_fixture {
		igt_stop_hang_detector();
		close(fd);
	}
}