pcie_num_lanes from the INFO ioctl does not reflect the PCIe lanes on the GPU side

pcie_mlw_mask indicates platform the PCIe lanes only and does not reflect the PCIe lanes on the GPU side.
This differs from document comments.

agd5f/linux@e3e84b0a

@mareko @agd5f

• Hardware
  • CPU: Ryzen 5 5600G (Max: PCIe Gen3x16)
  • GPU: Radeon RX 6600 (Max: PCIe Gen4x8, Available: PCIe Gen3x8)

drm_amdgpu_info_device {
    ...
    pcie_gen: 3,
    ...
    pcie_num_lanes: 16,
    ...
}
Marketing Name: [AMD Radeon RX 6600]
DeviceID.RevID: 0x73FF.0xC7

0001-drm-amdgpu-return-the-correct-PCIe-lanes-from-the-IN.patch

changed the description

The problem with pcie_bandwidth_available() is that it queries the current HW state which could be lower at any given time if the SMU firmware has dynamically adjust the gen or lanes once the driver is loaded. Probably better to cache the pcie state at init time in amdgpu_device_get_pcie_info() and use that for the IOCTL.

I have come up with a way to reflect the maximum lane width of GPU or upstream switch internal to dGPU in pcie_mlw_mask.

Or should we add a field for cache the PCIe state to amdgpu_dpm struct?

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 9095c05e0..ef1940448 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -6071,9 +6071,10 @@ static void amdgpu_device_partner_bandwidth(struct amdgpu_device *adev,
  */
 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
 {
-	struct pci_dev *pdev;
+	struct pci_dev *pdev, *us_pdev, *ds_pdev;
 	enum pci_bus_speed speed_cap, platform_speed_cap;
-	enum pcie_link_width platform_link_width;
+	enum pcie_link_width width_cap, platform_link_width;
+	uint32_t asic_width_cap_mask;
 
 	if (amdgpu_pcie_gen_cap)
 		adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
@@ -6155,6 +6156,68 @@ static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
 		}
 	}
 	if (adev->pm.pcie_mlw_mask == 0) {
+		/* asic caps */
+		pdev = adev->pdev;
+		ds_pdev = pci_upstream_bridge(pdev);
+
+		if (ds_pdev->vendor == PCI_VENDOR_ID_ATI) {
+			us_pdev = pci_upstream_bridge(ds_pdev);
+			width_cap = pcie_get_width_cap(us_pdev);
+		} else {
+			width_cap = pcie_get_width_cap(pdev);
+		}
+
+		if (width_cap == PCIE_LNK_WIDTH_UNKNOWN) {
+			asic_width_cap_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
+		} else {
+			switch (width_cap) {
+			case PCIE_LNK_X32:
+				asic_width_cap_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
+							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
+							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
+							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
+							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
+							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
+							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
+				break;
+			case PCIE_LNK_X16:
+				asic_width_cap_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
+							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
+							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
+							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
+							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
+							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
+				break;
+			case PCIE_LNK_X12:
+				asic_width_cap_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
+							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
+							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
+							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
+							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
+				break;
+			case PCIE_LNK_X8:
+				asic_width_cap_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
+							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
+							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
+							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
+				break;
+			case PCIE_LNK_X4:
+				asic_width_cap_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
+							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
+							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
+				break;
+			case PCIE_LNK_X2:
+				asic_width_cap_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
+							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
+				break;
+			case PCIE_LNK_X1:
+				asic_width_cap_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
+				break;
+			default:
+				break;
+			}
+		}
+		/* platform caps */
 		if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
 			adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
 		} else {
@@ -6205,6 +6268,7 @@ static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
 				break;
 			}
 		}
+		adev->pm.pcie_mlw_mask &= asic_width_cap_mask;
 	}
 }

How about something like this?

0001-drm-amdgpu-cache-gpu-pcie-link-width.patch

0002-drm-amdgpu-update-the-PCIe-lanes-reporting-in-the-IN.patch

Thank you for your support!
I had to replace adev->pm.pcie_mlw_mask = ... to adev->pm.pcie_mlw_mask |= ..., but it sure does work.

mentioned in commit agd5f/linux@9e424a5d

mentioned in commit agd5f/linux@6e55f87d

mentioned in commit agd5f/linux@8fae3b20

mentioned in commit agd5f/linux@58d0de42

mentioned in commit agd5f/linux@64314e3f

mentioned in commit agd5f/linux@757e8b95