diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c
index df23da5a36fd70f3f873bb3d2a02362f9380dfdb..dd4a4a6e0b94721cac2aa517278a5ad2e097aaa2 100644
--- a/drivers/gpu/drm/xe/xe_device.c
+++ b/drivers/gpu/drm/xe/xe_device.c
@@ -174,9 +174,7 @@ struct xe_device *xe_device_create(struct pci_dev *pdev,
 	struct xe_device *xe;
 	int err;
 
-	err = xe_display_set_driver_hooks(pdev, &driver);
-	if (err)
-		return ERR_PTR(err);
+	xe_display_driver_set_hooks(&driver);
 
 	err = drm_aperture_remove_conflicting_pci_framebuffers(pdev, &driver);
 	if (err)
diff --git a/drivers/gpu/drm/xe/xe_display.c b/drivers/gpu/drm/xe/xe_display.c
index f0c9fd41e42dde494f48b4010dad84efb0e564ba..1a9e9c78966d1794831295052c0d92961f9765a5 100644
--- a/drivers/gpu/drm/xe/xe_display.c
+++ b/drivers/gpu/drm/xe/xe_display.c
@@ -33,37 +33,47 @@
 
 /* Xe device functions */
 
+/**
+ * xe_display_driver_probe_defer - Detect if we need to wait for other drivers
+ *				   early on
+ * @pdev: PCI device
+ *
+ * Returns: 0 if probe can continue, -EPROBE_DEFER otherwise
+ */
+int xe_display_driver_probe_defer(struct pci_dev *pdev)
+{
+	if (!enable_display)
+		return 0;
+
+	return intel_display_driver_probe_defer(pdev);
+}
+
 static void xe_display_last_close(struct drm_device *dev)
 {
-	intel_fbdev_restore_mode(to_xe_device(dev));
+	struct xe_device *xe = to_xe_device(dev);
+
+	if (xe->info.enable_display)
+		intel_fbdev_restore_mode(to_xe_device(dev));
 }
 
 /**
- * xe_display_set_driver_hooks - set driver flags and hooks for display
- * @pdev: PCI device
+ * xe_display_driver_set_hooks - Add driver flags and hooks for display
  * @driver: DRM device driver
  *
  * Set features and function hooks in @driver that are needed for driving the
- * display IP, when that is enabled.
- *
- * Returns: 0 on success
+ * display IP. This sets the driver's capability of driving display, regardless
+ * if the device has it enabled
  */
-int xe_display_set_driver_hooks(struct pci_dev *pdev, struct drm_driver *driver)
+void xe_display_driver_set_hooks(struct drm_driver *driver)
 {
 	if (!enable_display)
-		return 0;
-
-	/* Detect if we need to wait for other drivers early on */
-	if (intel_display_driver_probe_defer(pdev))
-		return -EPROBE_DEFER;
+		return;
 
 	driver->driver_features |= DRIVER_MODESET | DRIVER_ATOMIC;
 	driver->lastclose = xe_display_last_close;
-
-	return 0;
 }
 
-static void unset_driver_hooks(struct xe_device *xe)
+static void unset_display_features(struct xe_device *xe)
 {
 	xe->drm.driver_features &= ~(DRIVER_MODESET | DRIVER_ATOMIC);
 }
@@ -482,7 +492,7 @@ __diag_ignore_all("-Woverride-init", "Allow field overrides in table");
 void xe_display_info_init(struct xe_device *xe)
 {
 	if (!xe->info.enable_display) {
-		unset_driver_hooks(xe);
+		unset_display_features(xe);
 		return;
 	}
 
@@ -534,7 +544,7 @@ void xe_display_info_init(struct xe_device *xe)
 	default:
 		drm_warn(&xe->drm, "Unknown display IP\n");
 		xe->info.enable_display = false;
-		unset_driver_hooks(xe);
+		unset_display_features(xe);
 		return;
 	}
 }
diff --git a/drivers/gpu/drm/xe/xe_display.h b/drivers/gpu/drm/xe/xe_display.h
index 84c556111c918b871f84a10f616a8f3101f66036..9e29de012df7c1ab3c86a3d440fe58558c12079c 100644
--- a/drivers/gpu/drm/xe/xe_display.h
+++ b/drivers/gpu/drm/xe/xe_display.h
@@ -12,7 +12,8 @@ struct drm_driver;
 
 #if IS_ENABLED(CONFIG_DRM_XE_DISPLAY)
 
-int xe_display_set_driver_hooks(struct pci_dev *pdev, struct drm_driver *driver);
+int xe_display_driver_probe_defer(struct pci_dev *pdev);
+void xe_display_driver_set_hooks(struct drm_driver *driver);
 
 int xe_display_create(struct xe_device *xe);
 
@@ -47,8 +48,9 @@ void xe_display_pm_resume(struct xe_device *xe);
 
 #else
 
-static inline int
-xe_display_set_driver_hooks(struct pci_dev *pdev, struct drm_driver *driver) { return 0; }
+static inline int xe_display_driver_probe_defer(struct pci_dev *pdev) { return 0; }
+
+static inline void xe_display_driver_set_hooks(struct drm_driver *driver) { }
 
 static inline int
 xe_display_create(struct xe_device *xe) { return 0; }
diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c
index 30a75a9cad65939cdf773eab283a0a5cb007f950..d8f1213ae1d6441923238dad89a9fbe90fd0ee46 100644
--- a/drivers/gpu/drm/xe/xe_pci.c
+++ b/drivers/gpu/drm/xe/xe_pci.c
@@ -638,6 +638,10 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		return -ENODEV;
 	}
 
+	err = xe_display_driver_probe_defer(pdev);
+	if (err)
+		return err;
+
 	xe = xe_device_create(pdev, ent);
 	if (IS_ERR(xe))
 		return PTR_ERR(xe);