From cfd1b7d42cd06d4d3417554bfda101a4baaaba09 Mon Sep 17 00:00:00 2001
From: Aaron Plattner <aplattner@nvidia.com>
Date: Fri, 22 Jan 2021 15:22:19 -0800
Subject: [PATCH] xfree86/modes: Use correct crtc gamma size in
 xf86RandR12InitGamma

When first initializing the DIX layer's RRCrtc structures, xf86RandR12Init12
hard-codes a gamma ramp size of 256 elements and then calls into the DDX layer
to copy the gamma ramps from the RRCrtc into the xf86Crtc's gamma ramp arrays.
When it does this, it assumes that crtc->gamma_size ==
crtc->randr_crtc->gammaSize == 256. However, commit 245b9db0 modified the
modesetting driver to update crtc->gamma_size with the true size of the
hardware's gamma ramp when the DRM GAMMA_LUT property is available. This causes
xf86RandR12CrtcSetGamma to read past the end of the
randr_crtc->gamma{Red,Green,Blue} arrays:

  PreInit
    drmmode_pre_init
      drmmode_crtc_init
        crtc->gamma_size = 1024

  ScreenInit
    xf86CrtcScreenInit
      xf86RandR12Init
        xf86RandR12Init12
          xf86RandR12CreateObjects12
            RRCrtcCreate
              randr_crtc->gammaSize = 0
          xf86RandR12InitGamma(pScrn, 256)
            RRCrtcGammaSetSize
              randr_crtc->gammaSize = 256
          xf86RandR12InitGamma
            xf86RandR12CrtcInitGamma
              RRCrtcGammaSet
                xf86RandR12CrtcSetGamma
                  // crtc->gamma_size is 1024 here, while randr_crtc->gammaRed
                  // is a 256-element array.
                  memcpy(crtc->gamma_red, randr_crtc->gammaRed, crtc->gamma_size * sizeof(crtc->gamma_red[0]));
    drmmode_setup_colormap
      xf86HandleColormaps
        xf86RandR12InitGamma
          RRCrtcGammaSetSize
            randr_crtc->gammaSize = 1024

Rather than passing gammaSize as an argument to xf86RandR12InitGamma, just use
the xf86Crtc's gamma size as the argument to RRCrtcGammaSetSize so that the
RRCrtc and xf86Crtc agree about the size of the gamma ramp immediately.

Fixes: 245b9db0 - modesetting: Use GAMMA_LUT when available
Closes: https://gitlab.freedesktop.org/xorg/xserver/-/issues/1126
Signed-off-by: Aaron Plattner <aplattner@nvidia.com>
---
 hw/xfree86/common/xf86cmap.c   | 2 +-
 hw/xfree86/modes/xf86RandR12.c | 6 +++---
 hw/xfree86/modes/xf86RandR12.h | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/hw/xfree86/common/xf86cmap.c b/hw/xfree86/common/xf86cmap.c
index 8588c9fba..28be5c2f8 100644
--- a/hw/xfree86/common/xf86cmap.c
+++ b/hw/xfree86/common/xf86cmap.c
@@ -228,7 +228,7 @@ xf86HandleColormaps(ScreenPtr pScreen,
     if (xf86_crtc_supports_gamma(pScrn)) {
         pScrn->LoadPalette = xf86RandR12LoadPalette;
 
-        if (!xf86RandR12InitGamma(pScrn, elements)) {
+        if (!xf86RandR12InitGamma(pScrn)) {
             CMapUnwrapScreen(pScreen);
             return FALSE;
         }
diff --git a/hw/xfree86/modes/xf86RandR12.c b/hw/xfree86/modes/xf86RandR12.c
index 50cbd043e..0e15edc57 100644
--- a/hw/xfree86/modes/xf86RandR12.c
+++ b/hw/xfree86/modes/xf86RandR12.c
@@ -1467,7 +1467,7 @@ xf86RandR12OutputInitGamma(xf86OutputPtr output)
 }
 
 Bool
-xf86RandR12InitGamma(ScrnInfoPtr pScrn, unsigned gammaSize) {
+xf86RandR12InitGamma(ScrnInfoPtr pScrn) {
     xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(pScrn);
     int o, c;
 
@@ -1480,7 +1480,7 @@ xf86RandR12InitGamma(ScrnInfoPtr pScrn, unsigned gammaSize) {
         if (!crtc->randr_crtc)
             continue;
 
-        if (!RRCrtcGammaSetSize(crtc->randr_crtc, gammaSize) ||
+        if (!RRCrtcGammaSetSize(crtc->randr_crtc, crtc->gamma_size) ||
             !xf86RandR12CrtcInitGamma(crtc, 1.0f, 1.0f, 1.0f))
             return FALSE;
     }
@@ -2379,7 +2379,7 @@ xf86RandR12Init12(ScreenPtr pScreen)
     if (!xf86RandR12SetInfo12(pScreen))
         return FALSE;
 
-    if (!xf86RandR12InitGamma(pScrn, 256))
+    if (!xf86RandR12InitGamma(pScrn))
         return FALSE;
 
     return TRUE;
diff --git a/hw/xfree86/modes/xf86RandR12.h b/hw/xfree86/modes/xf86RandR12.h
index 31aaaaf28..a3ea678be 100644
--- a/hw/xfree86/modes/xf86RandR12.h
+++ b/hw/xfree86/modes/xf86RandR12.h
@@ -43,6 +43,6 @@ extern _X_EXPORT void xf86RandR12TellChanged(ScreenPtr pScreen);
 extern void xf86RandR12LoadPalette(ScrnInfoPtr pScrn, int numColors,
                                    int *indices, LOCO *colors,
                                    VisualPtr pVisual);
-extern Bool xf86RandR12InitGamma(ScrnInfoPtr pScrn, unsigned gammaSize);
+extern Bool xf86RandR12InitGamma(ScrnInfoPtr pScrn);
 
 #endif                          /* _XF86_RANDR_H_ */
-- 
GitLab