deadlock in prepare_flip_isr/amdgpu_dm_commit_planes

I did a bisect and found out that this commit was causing trouble.

commit 446a98b19fd6da97a1fb148abb1766ad89c9b767 (HEAD)
Author: Thomas Gleixner <tglx@linutronix.de>
Date:   Thu Jul 29 23:51:58 2021 +0200

    PCI/MSI: Use new mask/unmask functions
    
    Switch the PCI/MSI core to use the new mask/unmask functions. No functional
    change.
    
    Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
    Tested-by: Marc Zyngier <maz@kernel.org>
    Reviewed-by: Marc Zyngier <maz@kernel.org>
    Link: https://lore.kernel.org/r/20210729222543.311207034@linutronix.de

diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
index 26dd91f33374..ce841f327ff6 100644
--- a/drivers/pci/msi.c
+++ b/drivers/pci/msi.c
@@ -135,74 +135,6 @@ void __weak arch_restore_msi_irqs(struct pci_dev *dev)
  * reliably as devices without an INTx disable bit will then generate a
  * level IRQ which will never be cleared.
  */
-static void __pci_msi_desc_mask_irq(struct msi_desc *desc, u32 mask, u32 flag)
-{
-       raw_spinlock_t *lock = &desc->dev->msi_lock;
-       unsigned long flags;
-
-       if (pci_msi_ignore_mask || !desc->msi_attrib.maskbit)
-               return;
-
-       raw_spin_lock_irqsave(lock, flags);
-       desc->msi_mask &= ~mask;
-       desc->msi_mask |= flag;
-       pci_write_config_dword(msi_desc_to_pci_dev(desc), desc->mask_pos,
-                              desc->msi_mask);
-       raw_spin_unlock_irqrestore(lock, flags);
-}
-
-static void msi_mask_irq(struct msi_desc *desc, u32 mask, u32 flag)
-{
-       __pci_msi_desc_mask_irq(desc, mask, flag);
-}
-
-static void __iomem *pci_msix_desc_addr(struct msi_desc *desc)
-{
-       return desc->mask_base + desc->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE;
-}
-
-/*
- * This internal function does not flush PCI writes to the device.
- * All users must ensure that they read from the device before either
- * assuming that the device state is up to date, or returning out of this
- * file.  This saves a few milliseconds when initialising devices with lots
- * of MSI-X interrupts.
- */
-static u32 __pci_msix_desc_mask_irq(struct msi_desc *desc, u32 flag)
-{
-       void __iomem *desc_addr = pci_msix_desc_addr(desc);
-       u32 ctrl = desc->msix_ctrl;
-
-       if (pci_msi_ignore_mask || desc->msi_attrib.is_virtual)
-               return 0;
-
-       ctrl &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT;
-       if (ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT)
-               ctrl |= PCI_MSIX_ENTRY_CTRL_MASKBIT;
-
-       writel(ctrl, desc_addr + PCI_MSIX_ENTRY_VECTOR_CTRL);
-
-       return ctrl;
-}
-
-static void msix_mask_irq(struct msi_desc *desc, u32 flag)
-{
-       desc->msix_ctrl = __pci_msix_desc_mask_irq(desc, flag);
-}
-
-static void msi_set_mask_bit(struct irq_data *data, u32 flag)
-{
-       struct msi_desc *desc = irq_data_get_msi_desc(data);
-
-       if (desc->msi_attrib.is_msix) {
-               msix_mask_irq(desc, flag);
-               readl(desc->mask_base);         /* Flush write to device */
-       } else {
-               unsigned offset = data->irq - desc->irq;
-               msi_mask_irq(desc, 1 << offset, flag << offset);
-       }
-}
-
 static inline __attribute_const__ u32 msi_multi_mask(struct msi_desc *desc)
 {
        /* Don't shift by >= width of type */
@@ -234,6 +166,11 @@ static inline void pci_msi_unmask(struct msi_desc *desc, u32 mask)
        pci_msi_update_mask(desc, mask, 0);
 }
 
+static inline void __iomem *pci_msix_desc_addr(struct msi_desc *desc)
+{
+       return desc->mask_base + desc->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE;
+}
+
 /*
  * This internal function does not flush PCI writes to the device.  All
  * users must ensure that they read from the device before either assuming
@@ -289,7 +226,9 @@ static void __pci_msi_unmask_desc(struct msi_desc *desc, u32 mask)
  */
 void pci_msi_mask_irq(struct irq_data *data)
 {
-       msi_set_mask_bit(data, 1);
+       struct msi_desc *desc = irq_data_get_msi_desc(data);
+
+       __pci_msi_mask_desc(desc, BIT(data->irq - desc->irq));
 }
 EXPORT_SYMBOL_GPL(pci_msi_mask_irq);
 
@@ -299,7 +238,9 @@ EXPORT_SYMBOL_GPL(pci_msi_mask_irq);
  */
 void pci_msi_unmask_irq(struct irq_data *data)
 {
-       msi_set_mask_bit(data, 0);
+       struct msi_desc *desc = irq_data_get_msi_desc(data);
+
+       __pci_msi_unmask_desc(desc, BIT(data->irq - desc->irq));
 }
 EXPORT_SYMBOL_GPL(pci_msi_unmask_irq);
 
@@ -352,7 +293,8 @@ void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
                /* Don't touch the hardware now */
        } else if (entry->msi_attrib.is_msix) {
                void __iomem *base = pci_msix_desc_addr(entry);
-               bool unmasked = !(entry->msix_ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT);
+               u32 ctrl = entry->msix_ctrl;
+               bool unmasked = !(ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT);
 
                if (entry->msi_attrib.is_virtual)
                        goto skip;
@@ -366,14 +308,14 @@ void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
                 * undefined."
                 */
                if (unmasked)
-                       __pci_msix_desc_mask_irq(entry, PCI_MSIX_ENTRY_CTRL_MASKBIT);
+                       pci_msix_write_vector_ctrl(entry, ctrl | PCI_MSIX_ENTRY_CTRL_MASKBIT);
 
                writel(msg->address_lo, base + PCI_MSIX_ENTRY_LOWER_ADDR);
                writel(msg->address_hi, base + PCI_MSIX_ENTRY_UPPER_ADDR);
                writel(msg->data, base + PCI_MSIX_ENTRY_DATA);
 
                if (unmasked)
-                       __pci_msix_desc_mask_irq(entry, 0);
+                       pci_msix_write_vector_ctrl(entry, ctrl);
 
                /* Ensure that the writes are visible in the device */
                readl(base + PCI_MSIX_ENTRY_DATA);
@@ -491,7 +433,7 @@ static void __pci_restore_msi_state(struct pci_dev *dev)
        arch_restore_msi_irqs(dev);
 
        pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control);
-       msi_mask_irq(entry, msi_multi_mask(entry), entry->msi_mask);
+       pci_msi_update_mask(entry, 0, 0);
        control &= ~PCI_MSI_FLAGS_QSIZE;
        control |= (entry->msi_attrib.multiple << 4) | PCI_MSI_FLAGS_ENABLE;
        pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control);
@@ -522,7 +464,7 @@ static void __pci_restore_msix_state(struct pci_dev *dev)
 
        arch_restore_msi_irqs(dev);
        for_each_pci_msi_entry(entry, dev)
-               msix_mask_irq(entry, entry->msix_ctrl);
+               pci_msix_write_vector_ctrl(entry, entry->msix_ctrl);
 
        pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0);
 }
@@ -704,7 +646,6 @@ static int msi_capability_init(struct pci_dev *dev, int nvec,
 {
        struct msi_desc *entry;
        int ret;
-       unsigned mask;
 
        pci_msi_set_enable(dev, 0);     /* Disable MSI during set up */
 
@@ -713,8 +654,7 @@ static int msi_capability_init(struct pci_dev *dev, int nvec,
                return -ENOMEM;
 
        /* All MSIs are unmasked by default; mask them all */
-       mask = msi_multi_mask(entry);
-       msi_mask_irq(entry, mask, mask);
+       pci_msi_mask(entry, msi_multi_mask(entry));
 
        list_add_tail(&entry->list, dev_to_msi_list(&dev->dev));
 
@@ -741,7 +681,7 @@ static int msi_capability_init(struct pci_dev *dev, int nvec,
        return 0;
 
 err:
-       msi_mask_irq(entry, mask, 0);
+       pci_msi_unmask(entry, msi_multi_mask(entry));
        free_msi_irqs(dev);
        return ret;
 }
@@ -1021,7 +961,7 @@ static void pci_msi_shutdown(struct pci_dev *dev)
        dev->msi_enabled = 0;
 
        /* Return the device with MSI unmasked as initial states */
-       msi_mask_irq(desc, msi_multi_mask(desc), 0);
+       pci_msi_unmask(desc, msi_multi_mask(desc));
 
        /* Restore dev->irq to its default pin-assertion IRQ */
        dev->irq = desc->msi_attrib.default_irq;
@@ -1107,7 +1047,7 @@ static void pci_msix_shutdown(struct pci_dev *dev)
 
        /* Return the device with MSI-X masked as initial states */
        for_each_pci_msi_entry(entry, dev)
-               __pci_msix_desc_mask_irq(entry, 1);
+               pci_msix_mask(entry);
 
        pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0);
        pci_intx_for_msi(dev, 1);

git bisect log

git bisect start
# new: [4357f03d6611753936e4d52fc251b54a6afb1b54] Merge tag 'pm-5.15-rc2' of git://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm
git bisect new 4357f03d6611753936e4d52fc251b54a6afb1b54
# old: [7d2a07b769330c34b4deabeed939325c77a7ec2f] Linux 5.14
git bisect old 7d2a07b769330c34b4deabeed939325c77a7ec2f
# new: [1b4f3dfb4792f03b139edf10124fcbeb44e608e6] Merge tag 'usb-serial-5.15-rc1' of https://git.kernel.org/pub/scm/linux/kernel/git/johan/usb-serial into usb-next
git bisect new 1b4f3dfb4792f03b139edf10124fcbeb44e608e6
# old: [29ce8f9701072fc221d9c38ad952de1a9578f95c] Merge git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net
git bisect old 29ce8f9701072fc221d9c38ad952de1a9578f95c
# new: [e7c1bbcf0c315c56cd970642214aa1df3d8cf61d] Merge tag 'hwmon-for-v5.15' of git://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging
git bisect new e7c1bbcf0c315c56cd970642214aa1df3d8cf61d
# new: [679369114e55f422dc593d0628cfde1d04ae59b3] Merge tag 'for-5.15/block-2021-08-30' of git://git.kernel.dk/linux-block
git bisect new 679369114e55f422dc593d0628cfde1d04ae59b3
# old: [c7a5238ef68b98130fe36716bb3fa44502f56001] Merge tag 's390-5.15-1' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux
git bisect old c7a5238ef68b98130fe36716bb3fa44502f56001
# old: [e5e726f7bb9f711102edea7e5bd511835640e3b4] Merge tag 'locking-core-2021-08-30' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
git bisect old e5e726f7bb9f711102edea7e5bd511835640e3b4
# old: [158ee7b65653d9f841823c249014c2d0dfdeeb8f] block: mark blkdev_fsync static
git bisect old 158ee7b65653d9f841823c249014c2d0dfdeeb8f
# new: [47fb0cfdb7a71a8a0ff8fe1d117363dc81f6ca77] Merge tag 'irqchip-5.15' of git://git.kernel.org/pub/scm/linux/kernel/git/maz/arm-platforms into irq/core
git bisect new 47fb0cfdb7a71a8a0ff8fe1d117363dc81f6ca77
# old: [4513fb87e1402ad815912ec7f027eb17149f44ee] Merge branch irq/misc-5.15 into irq/irqchip-next
git bisect old 4513fb87e1402ad815912ec7f027eb17149f44ee
# new: [88ffe2d0a55a165e55cedad1693f239d47e3e17e] genirq/cpuhotplug: Demote debug printk to KERN_DEBUG
git bisect new 88ffe2d0a55a165e55cedad1693f239d47e3e17e
# old: [fcacdfbef5a1633211ebfac1b669a7739f5b553e] PCI/MSI: Provide a new set of mask and unmask functions
git bisect old fcacdfbef5a1633211ebfac1b669a7739f5b553e
# new: [91cc470e797828d779cd4c1efbe8519bcb358bae] genirq: Change force_irqthreads to a static key
git bisect new 91cc470e797828d779cd4c1efbe8519bcb358bae
# new: [428e211641ed808b55cdc7d880a0ee349eff354b] genirq/affinity: Replace deprecated CPU-hotplug functions.
git bisect new 428e211641ed808b55cdc7d880a0ee349eff354b

related issue: https://github.com/QubesOS/qubes-issues/issues/6859

Works with reverting this patch and compiling latest mainline.

changed the description

Solved with kernel 5.15-rc2. Could be something else, but seems to be solved via update, will re-check.

I double checked my environment, it seems indeed that some revert upstream is making this go away.

Will try to bisect it, but currently in another bisect.

Back again with 5.15-rc4.

Kernel boots properly with pci=nomsi.

Can you bisect? Sounds like a core MSI change rather than something specific to the GPU driver.

Let me start by a fair bit of admiration of your hard work @agd5f. Woot!

The kernel still boots properly with 446a98b19fd6da97a1fb148abb1766ad89c9b767 reverted. But that's a bit harsh right? As it was working around -rc2 I should bisect between -rc2 and -rc3? I saw some commit reverted due to backlash there.

TBH I am a bit lost around all this MSI-stuff, never touched it before. I started doing my own patch-amdgpu-irq.c.patch to amdgpu_irq.c since it seems that it doesn't follow the MSI guidelines, I guess it doesn't change much though, but hey, I learned something today!

Could it be a USB driver and the amdgpu driver not playing along nicely?

My next try I will boot tip with PMC=y, and most likely just yank xhci/ehci from initramfs/modules. If that boots then I can be sure it's an IRQ problem between amdgpu and USB. I don't trust these chipsets that Lenovo sent along.

There's also the VT exception in the Xen log about Intel AX200, maybe just remove that one as well.

Starting a bisect now. Removing USB/Network etc from initramfs resulted in no change.

@agd5f I've a bit of trouble finding the working 5.15-rc2 commit, I was wondering, is there a way to make the kernel explode during the deadlock? Best would be after I unluck the system drive such that journald can print to the logs.

I did a bisect earlier and found PCI/MSI: Use new mask/unmask functions to be the culprit, is there any changes needing to be done due to the new mask/unmask functions? I try to look through other drivers changes but can't get any gotchas.

There's a XEN PV patch for MSI, but it does not fix this particular problem AFAIK.

I managed to print held locks during a lockup

irq-boot-debug.3.log

Not sure if that gives a hint?

Deadlock detection complained badly at swapper at the boot as well.

If reverting 446a98b19fd6da97a1fb148abb1766ad89c9b767 helps, you should contact Thomas Gleixner. He may have some ideas.

Well du'h. I contacted the list. Thanks for you infinite wisdom!

https://lore.kernel.org/linux-pci/b023adf9-e21c-59ac-de49-57915c8cede8@oderland.se/

For me in the future, just google the commit id that you reverted => find other peoples regression issues.

mentioned in issue #1722 (closed)

mentioned in issue #1717 (closed)

Solved by adding early exit in pci msi drivers. I'll add a correct patch here when it's in the kernel.

Thanks for the help!

closed

changed the description

deadlock in prepare_flip_isr/amdgpu_dm_commit_planes

Brief summary of the problem:

Hardware description:

System information:

How to reproduce the issue:

Attached files:

Related

traces

Designs

Child items ...

Activity

Admin message

Admin message

deadlock in prepare_flip_isr/amdgpu_dm_commit_planes

Brief summary of the problem:

Hardware description:

System information:

How to reproduce the issue:

Attached files:

Related

traces

Activity