amdgpu_device.c 92.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
/*
 * Copyright 2008 Advanced Micro Devices, Inc.
 * Copyright 2008 Red Hat Inc.
 * Copyright 2009 Jerome Glisse.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 * Authors: Dave Airlie
 *          Alex Deucher
 *          Jerome Glisse
 */
28
#include <linux/power_supply.h>
29
#include <linux/kthread.h>
30 31 32 33
#include <linux/console.h>
#include <linux/slab.h>
#include <drm/drmP.h>
#include <drm/drm_crtc_helper.h>
34
#include <drm/drm_atomic_helper.h>
35 36 37 38 39
#include <drm/amdgpu_drm.h>
#include <linux/vgaarb.h>
#include <linux/vga_switcheroo.h>
#include <linux/efi.h>
#include "amdgpu.h"
40
#include "amdgpu_trace.h"
41 42 43
#include "amdgpu_i2c.h"
#include "atom.h"
#include "amdgpu_atombios.h"
44
#include "amdgpu_atomfirmware.h"
45
#include "amd_pcie.h"
46 47 48
#ifdef CONFIG_DRM_AMDGPU_SI
#include "si.h"
#endif
49 50 51
#ifdef CONFIG_DRM_AMDGPU_CIK
#include "cik.h"
#endif
52
#include "vi.h"
53
#include "soc15.h"
54
#include "bif/bif_4_1_d.h"
55
#include <linux/pci.h>
56
#include <linux/firmware.h>
57
#include "amdgpu_vf_error.h"
58

59
#include "amdgpu_amdkfd.h"
60
#include "amdgpu_pm.h"
61

62
MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
63
MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
64
MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
65

66 67
#define AMDGPU_RESUME_MS		2000

68
static const char *amdgpu_asic_name[] = {
69 70 71 72 73
	"TAHITI",
	"PITCAIRN",
	"VERDE",
	"OLAND",
	"HAINAN",
74 75 76 77 78 79 80
	"BONAIRE",
	"KAVERI",
	"KABINI",
	"HAWAII",
	"MULLINS",
	"TOPAZ",
	"TONGA",
81
	"FIJI",
82
	"CARRIZO",
Samuel Li's avatar
Samuel Li committed
83
	"STONEY",
84 85
	"POLARIS10",
	"POLARIS11",
86
	"POLARIS12",
Leo Liu's avatar
Leo Liu committed
87
	"VEGAM",
Ken Wang's avatar
Ken Wang committed
88
	"VEGA10",
89
	"VEGA12",
90
	"VEGA20",
91
	"RAVEN",
92 93 94
	"LAST",
};

95 96
static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);

97 98 99 100 101 102 103 104
/**
 * amdgpu_device_is_px - Is the device is a dGPU with HG/PX power control
 *
 * @dev: drm_device pointer
 *
 * Returns true if the device is a dGPU with HG/PX power control,
 * otherwise return false.
 */
105 106 107 108
bool amdgpu_device_is_px(struct drm_device *dev)
{
	struct amdgpu_device *adev = dev->dev_private;

109
	if (adev->flags & AMD_IS_PX)
110 111 112 113 114 115 116
		return true;
	return false;
}

/*
 * MMIO register access helper functions.
 */
117 118 119 120 121 122 123 124 125
/**
 * amdgpu_mm_rreg - read a memory mapped IO register
 *
 * @adev: amdgpu_device pointer
 * @reg: dword aligned register offset
 * @acc_flags: access flags which require special behavior
 *
 * Returns the 32 bit value from the offset specified.
 */
126
uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
127
			uint32_t acc_flags)
128
{
129 130
	uint32_t ret;

131
	if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
132 133
		return amdgpu_virt_kiq_rreg(adev, reg);

134
	if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
135
		ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
136 137 138 139 140 141 142 143
	else {
		unsigned long flags;

		spin_lock_irqsave(&adev->mmio_idx_lock, flags);
		writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
		ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
		spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
	}
144 145
	trace_amdgpu_mm_rreg(adev->pdev->device, reg, ret);
	return ret;
146 147
}

148 149 150 151 152 153
/*
 * MMIO register read with bytes helper functions
 * @offset:bytes offset from MMIO start
 *
*/

154 155 156 157 158 159 160 161
/**
 * amdgpu_mm_rreg8 - read a memory mapped IO register
 *
 * @adev: amdgpu_device pointer
 * @offset: byte aligned register offset
 *
 * Returns the 8 bit value from the offset specified.
 */
162 163 164 165 166 167 168 169 170 171 172 173
uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset) {
	if (offset < adev->rmmio_size)
		return (readb(adev->rmmio + offset));
	BUG();
}

/*
 * MMIO register write with bytes helper functions
 * @offset:bytes offset from MMIO start
 * @value: the value want to be written to the register
 *
*/
174 175 176 177 178 179 180 181 182
/**
 * amdgpu_mm_wreg8 - read a memory mapped IO register
 *
 * @adev: amdgpu_device pointer
 * @offset: byte aligned register offset
 * @value: 8 bit value to write
 *
 * Writes the value specified to the offset specified.
 */
183 184 185 186 187 188 189
void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value) {
	if (offset < adev->rmmio_size)
		writeb(value, adev->rmmio + offset);
	else
		BUG();
}

190 191 192 193 194 195 196 197 198 199
/**
 * amdgpu_mm_wreg - write to a memory mapped IO register
 *
 * @adev: amdgpu_device pointer
 * @reg: dword aligned register offset
 * @v: 32 bit value to write to the register
 * @acc_flags: access flags which require special behavior
 *
 * Writes the value specified to the offset specified.
 */
200
void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
201
		    uint32_t acc_flags)
202
{
203
	trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
204

205 206 207 208
	if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
		adev->last_mm_index = v;
	}

209
	if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
210 211
		return amdgpu_virt_kiq_wreg(adev, reg, v);

212
	if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
213 214 215 216 217 218 219 220 221
		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
	else {
		unsigned long flags;

		spin_lock_irqsave(&adev->mmio_idx_lock, flags);
		writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
		writel(v, ((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
		spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
	}
222 223 224 225

	if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
		udelay(500);
	}
226 227
}

228 229 230 231 232 233 234 235
/**
 * amdgpu_io_rreg - read an IO register
 *
 * @adev: amdgpu_device pointer
 * @reg: dword aligned register offset
 *
 * Returns the 32 bit value from the offset specified.
 */
236 237 238 239 240 241 242 243 244 245
u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg)
{
	if ((reg * 4) < adev->rio_mem_size)
		return ioread32(adev->rio_mem + (reg * 4));
	else {
		iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
		return ioread32(adev->rio_mem + (mmMM_DATA * 4));
	}
}

246 247 248 249 250 251 252 253 254
/**
 * amdgpu_io_wreg - write to an IO register
 *
 * @adev: amdgpu_device pointer
 * @reg: dword aligned register offset
 * @v: 32 bit value to write to the register
 *
 * Writes the value specified to the offset specified.
 */
255 256
void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
{
257 258 259
	if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
		adev->last_mm_index = v;
	}
260 261 262 263 264 265 266

	if ((reg * 4) < adev->rio_mem_size)
		iowrite32(v, adev->rio_mem + (reg * 4));
	else {
		iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
		iowrite32(v, adev->rio_mem + (mmMM_DATA * 4));
	}
267 268 269 270

	if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
		udelay(500);
	}
271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310
}

/**
 * amdgpu_mm_rdoorbell - read a doorbell dword
 *
 * @adev: amdgpu_device pointer
 * @index: doorbell index
 *
 * Returns the value in the doorbell aperture at the
 * requested doorbell index (CIK).
 */
u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
{
	if (index < adev->doorbell.num_doorbells) {
		return readl(adev->doorbell.ptr + index);
	} else {
		DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
		return 0;
	}
}

/**
 * amdgpu_mm_wdoorbell - write a doorbell dword
 *
 * @adev: amdgpu_device pointer
 * @index: doorbell index
 * @v: value to write
 *
 * Writes @v to the doorbell aperture at the
 * requested doorbell index (CIK).
 */
void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
{
	if (index < adev->doorbell.num_doorbells) {
		writel(v, adev->doorbell.ptr + index);
	} else {
		DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
	}
}

311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348
/**
 * amdgpu_mm_rdoorbell64 - read a doorbell Qword
 *
 * @adev: amdgpu_device pointer
 * @index: doorbell index
 *
 * Returns the value in the doorbell aperture at the
 * requested doorbell index (VEGA10+).
 */
u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
{
	if (index < adev->doorbell.num_doorbells) {
		return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
	} else {
		DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
		return 0;
	}
}

/**
 * amdgpu_mm_wdoorbell64 - write a doorbell Qword
 *
 * @adev: amdgpu_device pointer
 * @index: doorbell index
 * @v: value to write
 *
 * Writes @v to the doorbell aperture at the
 * requested doorbell index (VEGA10+).
 */
void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
{
	if (index < adev->doorbell.num_doorbells) {
		atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
	} else {
		DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
	}
}

349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422
/**
 * amdgpu_invalid_rreg - dummy reg read function
 *
 * @adev: amdgpu device pointer
 * @reg: offset of register
 *
 * Dummy register read function.  Used for register blocks
 * that certain asics don't have (all asics).
 * Returns the value in the register.
 */
static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
{
	DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
	BUG();
	return 0;
}

/**
 * amdgpu_invalid_wreg - dummy reg write function
 *
 * @adev: amdgpu device pointer
 * @reg: offset of register
 * @v: value to write to the register
 *
 * Dummy register read function.  Used for register blocks
 * that certain asics don't have (all asics).
 */
static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
{
	DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
		  reg, v);
	BUG();
}

/**
 * amdgpu_block_invalid_rreg - dummy reg read function
 *
 * @adev: amdgpu device pointer
 * @block: offset of instance
 * @reg: offset of register
 *
 * Dummy register read function.  Used for register blocks
 * that certain asics don't have (all asics).
 * Returns the value in the register.
 */
static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
					  uint32_t block, uint32_t reg)
{
	DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
		  reg, block);
	BUG();
	return 0;
}

/**
 * amdgpu_block_invalid_wreg - dummy reg write function
 *
 * @adev: amdgpu device pointer
 * @block: offset of instance
 * @reg: offset of register
 * @v: value to write to the register
 *
 * Dummy register read function.  Used for register blocks
 * that certain asics don't have (all asics).
 */
static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
				      uint32_t block,
				      uint32_t reg, uint32_t v)
{
	DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
		  reg, block, v);
	BUG();
}

423 424 425 426 427 428 429 430
/**
 * amdgpu_device_vram_scratch_init - allocate the VRAM scratch page
 *
 * @adev: amdgpu device pointer
 *
 * Allocates a scratch page of VRAM for use by various things in the
 * driver.
 */
431
static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev)
432
{
433 434 435 436 437
	return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE,
				       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
				       &adev->vram_scratch.robj,
				       &adev->vram_scratch.gpu_addr,
				       (void **)&adev->vram_scratch.ptr);
438 439
}

440 441 442 443 444 445 446
/**
 * amdgpu_device_vram_scratch_fini - Free the VRAM scratch page
 *
 * @adev: amdgpu device pointer
 *
 * Frees the VRAM scratch page.
 */
447
static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev)
448
{
449
	amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL);
450 451 452
}

/**
453
 * amdgpu_device_program_register_sequence - program an array of registers.
454 455 456 457 458 459 460 461
 *
 * @adev: amdgpu_device pointer
 * @registers: pointer to the register array
 * @array_size: size of the register array
 *
 * Programs an array or registers with and and or masks.
 * This is a helper for setting golden registers.
 */
462 463 464
void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
					     const u32 *registers,
					     const u32 array_size)
465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487
{
	u32 tmp, reg, and_mask, or_mask;
	int i;

	if (array_size % 3)
		return;

	for (i = 0; i < array_size; i +=3) {
		reg = registers[i + 0];
		and_mask = registers[i + 1];
		or_mask = registers[i + 2];

		if (and_mask == 0xffffffff) {
			tmp = or_mask;
		} else {
			tmp = RREG32(reg);
			tmp &= ~and_mask;
			tmp |= or_mask;
		}
		WREG32(reg, tmp);
	}
}

488 489 490 491 492 493 494 495
/**
 * amdgpu_device_pci_config_reset - reset the GPU
 *
 * @adev: amdgpu_device pointer
 *
 * Resets the GPU using the pci config reset sequence.
 * Only applicable to asics prior to vega10.
 */
496
void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
497 498 499 500 501 502 503 504
{
	pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
}

/*
 * GPU doorbell aperture helpers function.
 */
/**
505
 * amdgpu_device_doorbell_init - Init doorbell driver information.
506 507 508 509 510 511
 *
 * @adev: amdgpu_device pointer
 *
 * Init doorbell driver information (CIK)
 * Returns 0 on success, error on failure.
 */
512
static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
513
{
514 515 516 517 518 519 520 521 522
	/* No doorbell on SI hardware generation */
	if (adev->asic_type < CHIP_BONAIRE) {
		adev->doorbell.base = 0;
		adev->doorbell.size = 0;
		adev->doorbell.num_doorbells = 0;
		adev->doorbell.ptr = NULL;
		return 0;
	}

523 524 525
	if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
		return -EINVAL;

526 527 528 529
	/* doorbell bar mapping */
	adev->doorbell.base = pci_resource_start(adev->pdev, 2);
	adev->doorbell.size = pci_resource_len(adev->pdev, 2);

530
	adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),
531 532 533 534
					     AMDGPU_DOORBELL_MAX_ASSIGNMENT+1);
	if (adev->doorbell.num_doorbells == 0)
		return -EINVAL;

535 536 537 538
	adev->doorbell.ptr = ioremap(adev->doorbell.base,
				     adev->doorbell.num_doorbells *
				     sizeof(u32));
	if (adev->doorbell.ptr == NULL)
539 540 541 542 543 544
		return -ENOMEM;

	return 0;
}

/**
545
 * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
546 547 548 549 550
 *
 * @adev: amdgpu_device pointer
 *
 * Tear down doorbell driver information (CIK)
 */
551
static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
552 553 554 555 556
{
	iounmap(adev->doorbell.ptr);
	adev->doorbell.ptr = NULL;
}

557

558 559

/*
560
 * amdgpu_device_wb_*()
561
 * Writeback is the method by which the GPU updates special pages in memory
Alex Xie's avatar
Alex Xie committed
562
 * with the status of certain GPU events (fences, ring pointers,etc.).
563 564 565
 */

/**
566
 * amdgpu_device_wb_fini - Disable Writeback and free memory
567 568 569 570 571 572
 *
 * @adev: amdgpu_device pointer
 *
 * Disables Writeback and frees the Writeback memory (all asics).
 * Used at driver shutdown.
 */
573
static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
574 575
{
	if (adev->wb.wb_obj) {
576 577 578
		amdgpu_bo_free_kernel(&adev->wb.wb_obj,
				      &adev->wb.gpu_addr,
				      (void **)&adev->wb.wb);
579 580 581 582 583
		adev->wb.wb_obj = NULL;
	}
}

/**
584
 * amdgpu_device_wb_init- Init Writeback driver info and allocate memory
585 586 587
 *
 * @adev: amdgpu_device pointer
 *
588
 * Initializes writeback and allocates writeback memory (all asics).
589 590 591
 * Used at driver startup.
 * Returns 0 on success or an -error on failure.
 */
592
static int amdgpu_device_wb_init(struct amdgpu_device *adev)
593 594 595 596
{
	int r;

	if (adev->wb.wb_obj == NULL) {
597 598
		/* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
		r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
599 600 601
					    PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
					    &adev->wb.wb_obj, &adev->wb.gpu_addr,
					    (void **)&adev->wb.wb);
602 603 604 605 606 607 608 609 610
		if (r) {
			dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
			return r;
		}

		adev->wb.num_wb = AMDGPU_MAX_WB;
		memset(&adev->wb.used, 0, sizeof(adev->wb.used));

		/* clear wb memory */
611
		memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
612 613 614 615 616 617
	}

	return 0;
}

/**
618
 * amdgpu_device_wb_get - Allocate a wb entry
619 620 621 622 623 624 625
 *
 * @adev: amdgpu_device pointer
 * @wb: wb index
 *
 * Allocate a wb slot for use by the driver (all asics).
 * Returns 0 on success or -EINVAL on failure.
 */
626
int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
627 628 629
{
	unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);

630
	if (offset < adev->wb.num_wb) {
Ken Wang's avatar
Ken Wang committed
631
		__set_bit(offset, adev->wb.used);
Monk Liu's avatar
Monk Liu committed
632
		*wb = offset << 3; /* convert to dw offset */
633 634 635 636 637 638
		return 0;
	} else {
		return -EINVAL;
	}
}

639
/**
640
 * amdgpu_device_wb_free - Free a wb entry
641 642 643 644 645 646
 *
 * @adev: amdgpu_device pointer
 * @wb: wb index
 *
 * Free a wb slot allocated for use by the driver (all asics)
 */
647
void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
648
{
649
	wb >>= 3;
650
	if (wb < adev->wb.num_wb)
651
		__clear_bit(wb, adev->wb.used);
652 653 654
}

/**
655
 * amdgpu_device_vram_location - try to find VRAM location
656
 *
657 658 659 660
 * @adev: amdgpu device structure holding all necessary informations
 * @mc: memory controller structure holding memory informations
 * @base: base address at which to put VRAM
 *
661
 * Function will try to place VRAM at base address provided
662
 * as parameter.
663
 */
664
void amdgpu_device_vram_location(struct amdgpu_device *adev,
665
				 struct amdgpu_gmc *mc, u64 base)
666 667 668 669 670 671 672 673 674 675 676 677 678
{
	uint64_t limit = (uint64_t)amdgpu_vram_limit << 20;

	mc->vram_start = base;
	mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
	if (limit && limit < mc->real_vram_size)
		mc->real_vram_size = limit;
	dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
			mc->mc_vram_size >> 20, mc->vram_start,
			mc->vram_end, mc->real_vram_size >> 20);
}

/**
679
 * amdgpu_device_gart_location - try to find GART location
680
 *
681 682 683
 * @adev: amdgpu device structure holding all necessary informations
 * @mc: memory controller structure holding memory informations
 *
684
 * Function will place try to place GART before or after VRAM.
685
 *
686
 * If GART size is bigger than space left then we ajust GART size.
687 688
 * Thus function will never fails.
 */
689
void amdgpu_device_gart_location(struct amdgpu_device *adev,
690
				 struct amdgpu_gmc *mc)
691 692 693
{
	u64 size_af, size_bf;

694 695
	mc->gart_size += adev->pm.smu_prv_buffer_size;

696
	size_af = adev->gmc.mc_mask - mc->vram_end;
697
	size_bf = mc->vram_start;
698
	if (size_bf > size_af) {
699
		if (mc->gart_size > size_bf) {
700
			dev_warn(adev->dev, "limiting GART\n");
701
			mc->gart_size = size_bf;
702
		}
703
		mc->gart_start = 0;
704
	} else {
705
		if (mc->gart_size > size_af) {
706
			dev_warn(adev->dev, "limiting GART\n");
707
			mc->gart_size = size_af;
708
		}
709 710 711 712
		/* VCE doesn't like it when BOs cross a 4GB segment, so align
		 * the GART base on a 4GB boundary as well.
		 */
		mc->gart_start = ALIGN(mc->vram_end + 1, 0x100000000ULL);
713
	}
714
	mc->gart_end = mc->gart_start + mc->gart_size - 1;
715
	dev_info(adev->dev, "GART: %lluM 0x%016llX - 0x%016llX\n",
716
			mc->gart_size >> 20, mc->gart_start, mc->gart_end);
717 718
}

719 720 721 722 723 724 725 726 727 728 729
/**
 * amdgpu_device_resize_fb_bar - try to resize FB BAR
 *
 * @adev: amdgpu_device pointer
 *
 * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
 * to fail, but if any of the BARs is not accessible after the size we abort
 * driver loading by returning -ENODEV.
 */
int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
{
730
	u64 space_needed = roundup_pow_of_two(adev->gmc.real_vram_size);
731
	u32 rbar_size = order_base_2(((space_needed >> 20) | 1)) - 1;
732 733 734
	struct pci_bus *root;
	struct resource *res;
	unsigned i;
735 736 737
	u16 cmd;
	int r;

738 739 740 741
	/* Bypass for VF */
	if (amdgpu_sriov_vf(adev))
		return 0;

742 743 744 745 746 747
	/* Check if the root BUS has 64bit memory resources */
	root = adev->pdev->bus;
	while (root->parent)
		root = root->parent;

	pci_bus_for_each_resource(root, res, i) {
748
		if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
749 750 751 752 753 754 755 756
		    res->start > 0x100000000ull)
			break;
	}

	/* Trying to resize is pointless without a root hub window above 4GB */
	if (!res)
		return 0;

757 758 759 760 761 762
	/* Disable memory decoding while we change the BAR addresses and size */
	pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
	pci_write_config_word(adev->pdev, PCI_COMMAND,
			      cmd & ~PCI_COMMAND_MEMORY);

	/* Free the VRAM and doorbell BAR, we most likely need to move both. */
763
	amdgpu_device_doorbell_fini(adev);
764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779
	if (adev->asic_type >= CHIP_BONAIRE)
		pci_release_resource(adev->pdev, 2);

	pci_release_resource(adev->pdev, 0);

	r = pci_resize_resource(adev->pdev, 0, rbar_size);
	if (r == -ENOSPC)
		DRM_INFO("Not enough PCI address space for a large BAR.");
	else if (r && r != -ENOTSUPP)
		DRM_ERROR("Problem resizing BAR0 (%d).", r);

	pci_assign_unassigned_bus_resources(adev->pdev->bus);

	/* When the doorbell or fb BAR isn't available we have no chance of
	 * using the device.
	 */
780
	r = amdgpu_device_doorbell_init(adev);
781 782 783 784 785 786 787
	if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
		return -ENODEV;

	pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);

	return 0;
}
788

789 790 791 792
/*
 * GPU helpers function.
 */
/**
793
 * amdgpu_device_need_post - check if the hw need post or not
794 795 796
 *
 * @adev: amdgpu_device pointer
 *
797 798 799
 * Check if the asic has been initialized (all asics) at driver startup
 * or post is needed if  hw reset is performed.
 * Returns true if need or false if not.
800
 */
801
bool amdgpu_device_need_post(struct amdgpu_device *adev)
802 803 804
{
	uint32_t reg;

805 806 807 808
	if (amdgpu_sriov_vf(adev))
		return false;

	if (amdgpu_passthrough(adev)) {
Monk Liu's avatar
Monk Liu committed
809 810 811 812
		/* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
		 * some old smc fw still need driver do vPost otherwise gpu hang, while
		 * those smc fw version above 22.15 doesn't have this flaw, so we force
		 * vpost executed for smc version below 22.15
813 814 815 816 817 818 819 820 821 822
		 */
		if (adev->asic_type == CHIP_FIJI) {
			int err;
			uint32_t fw_ver;
			err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
			/* force vPost if error occured */
			if (err)
				return true;

			fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
Monk Liu's avatar
Monk Liu committed
823 824
			if (fw_ver < 0x00160e00)
				return true;
825 826
		}
	}
827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843

	if (adev->has_hw_reset) {
		adev->has_hw_reset = false;
		return true;
	}

	/* bios scratch used on CIK+ */
	if (adev->asic_type >= CHIP_BONAIRE)
		return amdgpu_atombios_scratch_need_asic_init(adev);

	/* check MEM_SIZE for older asics */
	reg = amdgpu_asic_get_config_memsize(adev);

	if ((reg != 0) && (reg != 0xffffffff))
		return false;

	return true;
844 845
}

846 847
/* if we get transitioned to only one device, take VGA back */
/**
848
 * amdgpu_device_vga_set_decode - enable/disable vga decode
849 850 851 852 853 854 855
 *
 * @cookie: amdgpu_device pointer
 * @state: enable/disable vga decode
 *
 * Enable/disable vga decode (all asics).
 * Returns VGA resource flags.
 */
856
static unsigned int amdgpu_device_vga_set_decode(void *cookie, bool state)
857 858 859 860 861 862 863 864 865 866
{
	struct amdgpu_device *adev = cookie;
	amdgpu_asic_set_vga_state(adev, state);
	if (state)
		return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
		       VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
	else
		return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
}

867 868 869 870 871 872 873 874 875 876
/**
 * amdgpu_device_check_block_size - validate the vm block size
 *
 * @adev: amdgpu_device pointer
 *
 * Validates the vm block size specified via module parameter.
 * The vm block size defines number of bits in page table versus page directory,
 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
 * page table and the remaining bits are in the page directory.
 */
877
static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
878 879 880 881
{
	/* defines number of bits in page table versus page directory,
	 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
	 * page table and the remaining bits are in the page directory */
882 883
	if (amdgpu_vm_block_size == -1)
		return;
884

885
	if (amdgpu_vm_block_size < 9) {
886 887
		dev_warn(adev->dev, "VM page table size (%d) too small\n",
			 amdgpu_vm_block_size);
888
		amdgpu_vm_block_size = -1;
889 890 891
	}
}

892 893 894 895 896 897 898 899
/**
 * amdgpu_device_check_vm_size - validate the vm size
 *
 * @adev: amdgpu_device pointer
 *
 * Validates the vm size in GB specified via module parameter.
 * The VM size is the size of the GPU virtual memory space in GB.
 */
900
static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
901
{
902 903 904 905
	/* no need to check the default value */
	if (amdgpu_vm_size == -1)
		return;

906 907 908
	if (amdgpu_vm_size < 1) {
		dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
			 amdgpu_vm_size);
909
		amdgpu_vm_size = -1;
910 911 912
	}
}

913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952
static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
{
	struct sysinfo si;
	bool is_os_64 = (sizeof(void *) == 8) ? true : false;
	uint64_t total_memory;
	uint64_t dram_size_seven_GB = 0x1B8000000;
	uint64_t dram_size_three_GB = 0xB8000000;

	if (amdgpu_smu_memory_pool_size == 0)
		return;

	if (!is_os_64) {
		DRM_WARN("Not 64-bit OS, feature not supported\n");
		goto def_value;
	}
	si_meminfo(&si);
	total_memory = (uint64_t)si.totalram * si.mem_unit;

	if ((amdgpu_smu_memory_pool_size == 1) ||
		(amdgpu_smu_memory_pool_size == 2)) {
		if (total_memory < dram_size_three_GB)
			goto def_value1;
	} else if ((amdgpu_smu_memory_pool_size == 4) ||
		(amdgpu_smu_memory_pool_size == 8)) {
		if (total_memory < dram_size_seven_GB)
			goto def_value1;
	} else {
		DRM_WARN("Smu memory pool size not supported\n");
		goto def_value;
	}
	adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;

	return;

def_value1:
	DRM_WARN("No enough system memory\n");
def_value:
	adev->pm.smu_prv_buffer_size = 0;
}

953
/**
954
 * amdgpu_device_check_arguments - validate module params
955 956 957 958 959 960
 *
 * @adev: amdgpu_device pointer
 *
 * Validates certain module parameters and updates
 * the associated values used by the driver (all asics).
 */
961
static void amdgpu_device_check_arguments(struct amdgpu_device *adev)
962
{
963 964 965 966
	if (amdgpu_sched_jobs < 4) {
		dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
			 amdgpu_sched_jobs);
		amdgpu_sched_jobs = 4;
967
	} else if (!is_power_of_2(amdgpu_sched_jobs)){
968 969 970 971
		dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
			 amdgpu_sched_jobs);
		amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
	}
972

973
	if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
974 975 976
		/* gart size must be greater or equal to 32M */
		dev_warn(adev->dev, "gart size (%d) too small\n",
			 amdgpu_gart_size);
977
		amdgpu_gart_size = -1;
978 979
	}

980
	if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
981
		/* gtt size must be greater or equal to 32M */
982 983 984
		dev_warn(adev->dev, "gtt size (%d) too small\n",
				 amdgpu_gtt_size);
		amdgpu_gtt_size = -1;
985 986
	}

987 988 989 990 991 992 993
	/* valid range is between 4 and 9 inclusive */
	if (amdgpu_vm_fragment_size != -1 &&
	    (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
		dev_warn(adev->dev, "valid range is between 4 and 9\n");
		amdgpu_vm_fragment_size = -1;
	}

994 995
	amdgpu_device_check_smu_prv_buffer_size(adev);

996
	amdgpu_device_check_vm_size(adev);
997

998
	amdgpu_device_check_block_size(adev);
999

1000
	if (amdgpu_vram_page_split != -1 && (amdgpu_vram_page_split < 16 ||
1001
	    !is_power_of_2(amdgpu_vram_page_split))) {
1002 1003 1004 1005
		dev_warn(adev->dev, "invalid VRAM page split (%d)\n",
			 amdgpu_vram_page_split);
		amdgpu_vram_page_split = 1024;
	}
1006 1007 1008 1009 1010

	if (amdgpu_lockup_timeout == 0) {
		dev_warn(adev->dev, "lockup_timeout msut be > 0, adjusting to 10000\n");
		amdgpu_lockup_timeout = 10000;
	}
1011 1012

	adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1013 1014 1015 1016 1017 1018
}

/**
 * amdgpu_switcheroo_set_state - set switcheroo state
 *
 * @pdev: pci dev pointer
1019
 * @state: vga_switcheroo state
1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031
 *
 * Callback for the switcheroo driver.  Suspends or resumes the
 * the asics before or after it is powered up using ACPI methods.
 */
static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
{
	struct drm_device *dev = pci_get_drvdata(pdev);

	if (amdgpu_device_is_px(dev) && state == VGA_SWITCHEROO_OFF)
		return;

	if (state == VGA_SWITCHEROO_ON) {
1032
		pr_info("amdgpu: switched on\n");
1033 1034 1035
		/* don't suspend or resume card normally */
		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;

1036
		amdgpu_device_resume(dev, true, true);
1037 1038 1039 1040

		dev->switch_power_state = DRM_SWITCH_POWER_ON;
		drm_kms_helper_poll_enable(dev);
	} else {
1041
		pr_info("amdgpu: switched off\n");
1042 1043
		drm_kms_helper_poll_disable(dev);
		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1044
		amdgpu_device_suspend(dev, true, true);
1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075
		dev->switch_power_state = DRM_SWITCH_POWER_OFF;
	}
}

/**
 * amdgpu_switcheroo_can_switch - see if switcheroo state can change
 *
 * @pdev: pci dev pointer
 *
 * Callback for the switcheroo driver.  Check of the switcheroo
 * state can be changed.
 * Returns true if the state can be changed, false if not.
 */
static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
{
	struct drm_device *dev = pci_get_drvdata(pdev);

	/*
	* FIXME: open_count is protected by drm_global_mutex but that would lead to
	* locking inversion with the driver load path. And the access here is
	* completely racy anyway. So don't bother with locking for now.
	*/
	return dev->open_count == 0;
}

static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
	.set_gpu_state = amdgpu_switcheroo_set_state,
	.reprobe = NULL,
	.can_switch = amdgpu_switcheroo_can_switch,
};

1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086
/**
 * amdgpu_device_ip_set_clockgating_state - set the CG state
 *
 * @adev: amdgpu_device pointer
 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
 * @state: clockgating state (gate or ungate)
 *
 * Sets the requested clockgating state for all instances of
 * the hardware IP specified.
 * Returns the error code from the last instance.
 */
1087
int amdgpu_device_ip_set_clockgating_state(void *dev,
1088 1089
					   enum amd_ip_block_type block_type,
					   enum amd_clockgating_state state)
1090
{
1091
	struct amdgpu_device *adev = dev;
1092 1093 1094
	int i, r = 0;

	for (i = 0; i < adev->num_ip_blocks; i++) {
1095
		if (!adev->ip_blocks[i].status.valid)
1096
			continue;
1097 1098 1099 1100 1101 1102 1103 1104 1105
		if (adev->ip_blocks[i].version->type != block_type)
			continue;
		if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
			continue;
		r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
			(void *)adev, state);
		if (r)
			DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
				  adev->ip_blocks[i].version->funcs->name, r);
1106 1107 1108 1109
	}
	return r;
}

1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120
/**
 * amdgpu_device_ip_set_powergating_state - set the PG state
 *
 * @adev: amdgpu_device pointer
 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
 * @state: powergating state (gate or ungate)
 *
 * Sets the requested powergating state for all instances of
 * the hardware IP specified.
 * Returns the error code from the last instance.
 */
1121
int amdgpu_device_ip_set_powergating_state(void *dev,
1122 1123
					   enum amd_ip_block_type block_type,
					   enum amd_powergating_state state)
1124
{
1125
	struct amdgpu_device *adev = dev;
1126 1127 1128
	int i, r = 0;

	for (i = 0; i < adev->num_ip_blocks; i++) {
1129
		if (!adev->ip_blocks[i].status.valid)
1130
			continue;
1131 1132 1133 1134 1135 1136 1137 1138 1139
		if (adev->ip_blocks[i].version->type != block_type)
			continue;
		if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
			continue;
		r = adev->ip_blocks[i].version->funcs->set_powergating_state(
			(void *)adev, state);
		if (r)
			DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
				  adev->ip_blocks[i].version->funcs->name, r);
1140 1141 1142 1143
	}
	return r;
}

1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154
/**
 * amdgpu_device_ip_get_clockgating_state - get the CG state
 *
 * @adev: amdgpu_device pointer
 * @flags: clockgating feature flags
 *
 * Walks the list of IPs on the device and updates the clockgating
 * flags for each IP.
 * Updates @flags with the feature flags for each hardware IP where
 * clockgating is enabled.
 */
1155 1156
void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
					    u32 *flags)
1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167
{
	int i;

	for (i = 0; i < adev->num_ip_blocks; i++) {
		if (!adev->ip_blocks[i].status.valid)
			continue;
		if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
			adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
	}
}

1168 1169 1170 1171 1172 1173 1174 1175 1176
/**
 * amdgpu_device_ip_wait_for_idle - wait for idle
 *
 * @adev: amdgpu_device pointer
 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
 *
 * Waits for the request hardware IP to be idle.
 * Returns 0 for success or a negative error code on failure.
 */
1177 1178
int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
				   enum amd_ip_block_type block_type)
1179 1180 1181 1182
{
	int i, r;

	for (i = 0; i < adev->num_ip_blocks; i++) {
1183
		if (!adev->ip_blocks[i].status.valid)
1184
			continue;
1185 1186
		if (adev->ip_blocks[i].version->type == block_type) {
			r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1187 1188 1189 1190 1191 1192 1193 1194 1195
			if (r)
				return r;
			break;
		}
	}
	return 0;

}

1196 1197 1198 1199 1200 1201 1202 1203 1204
/**
 * amdgpu_device_ip_is_idle - is the hardware IP idle
 *
 * @adev: amdgpu_device pointer
 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
 *
 * Check if the hardware IP is idle or not.
 * Returns true if it the IP is idle, false if not.
 */
1205 1206
bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
			      enum amd_ip_block_type block_type)
1207 1208 1209 1210
{
	int i;

	for (i = 0; i < adev->num_ip_blocks; i++) {
1211
		if (!adev->ip_blocks[i].status.valid)
1212
			continue;
1213 1214
		if (adev->ip_blocks[i].version->type == block_type)
			return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1215 1216 1217 1218 1219
	}
	return true;

}

1220 1221 1222 1223 1224 1225 1226 1227 1228
/**
 * amdgpu_device_ip_get_ip_block - get a hw IP pointer
 *
 * @adev: amdgpu_device pointer
 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
 *
 * Returns a pointer to the hardware IP block structure
 * if it exists for the asic, otherwise NULL.
 */
1229 1230 1231
struct amdgpu_ip_block *
amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
			      enum amd_ip_block_type type)
1232 1233 1234 1235
{
	int i;

	for (i = 0; i < adev->num_ip_blocks; i++)
1236
		if (adev->ip_blocks[i].version->type == type)
1237 1238 1239 1240 1241 1242
			return &adev->ip_blocks[i];

	return NULL;
}

/**
1243
 * amdgpu_device_ip_block_version_cmp
1244 1245
 *
 * @adev: amdgpu_device pointer
1246
 * @type: enum amd_ip_block_type
1247 1248 1249 1250 1251 1252
 * @major: major version
 * @minor: minor version
 *
 * return 0 if equal or greater
 * return 1 if smaller or the ip_block doesn't exist
 */
1253 1254 1255
int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
				       enum amd_ip_block_type type,
				       u32 major, u32 minor)
1256
{
1257
	struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
1258

1259 1260 1261
	if (ip_block && ((ip_block->version->major > major) ||
			((ip_block->version->major == major) &&
			(ip_block->version->minor >= minor))))