Handle transform failure when computing shadow damage area.

PictureTransformBounds can fail, when this happens, damage the entire screen
so that the shadow gets repainted correctly.

hw/xfree86/modes/xf86Rotate.c

@@ -134,25 +134,6 @@ xf86RotateCrtcRedisplay (xf86CrtcPtr crtc, RegionPtr region)
     FreePicture (dst, None);
 }
 
-static void
-xf86CrtcDamageShadow (xf86CrtcPtr crtc)
-{
-    ScrnInfoPtr	pScrn = crtc->scrn;
-    BoxRec	damage_box;
-    RegionRec   damage_region;
-    ScreenPtr	pScreen = pScrn->pScreen;
-
-    damage_box.x1 = 0;
-    damage_box.x2 = crtc->mode.HDisplay;
-    damage_box.y1 = 0;
-    damage_box.y2 = crtc->mode.VDisplay;
-    PictureTransformBounds (&damage_box, &crtc->crtc_to_framebuffer);
-    REGION_INIT (pScreen, &damage_region, &damage_box, 1);
-    DamageRegionAppend(&(*pScreen->GetScreenPixmap)(pScreen)->drawable,
-			&damage_region);
-    REGION_UNINIT (pScreen, &damage_region);
-}
-
 static void
 xf86CrtcShadowClear (xf86CrtcPtr crtc)
 {
@@ -184,6 +165,36 @@ xf86CrtcShadowClear (xf86CrtcPtr crtc)
     FreePicture (dst, None);
 }
 
+static void
+xf86CrtcDamageShadow (xf86CrtcPtr crtc)
+{
+    ScrnInfoPtr	pScrn = crtc->scrn;
+    BoxRec	damage_box;
+    RegionRec   damage_region;
+    ScreenPtr	pScreen = pScrn->pScreen;
+
+    damage_box.x1 = 0;
+    damage_box.x2 = crtc->mode.HDisplay;
+    damage_box.y1 = 0;
+    damage_box.y2 = crtc->mode.VDisplay;
+    if (!PictureTransformBounds (&damage_box, &crtc->crtc_to_framebuffer))
+    {
+	damage_box.x1 = 0;
+	damage_box.y1 = 0;
+	damage_box.x2 = pScreen->width;
+	damage_box.y2 = pScreen->height;
+    }
+    if (damage_box.x1 < 0) damage_box.x1 = 0;
+    if (damage_box.y1 < 0) damage_box.y1 = 0;
+    if (damage_box.x2 > pScreen->width) damage_box.x2 = pScreen->width;
+    if (damage_box.y2 > pScreen->height) damage_box.y2 = pScreen->height;
+    REGION_INIT (pScreen, &damage_region, &damage_box, 1);
+    DamageRegionAppend (&(*pScreen->GetScreenPixmap)(pScreen)->drawable,
+			&damage_region);
+    REGION_UNINIT (pScreen, &damage_region);
+    xf86CrtcShadowClear (crtc);
+}
+
 static void
 xf86RotatePrepare (ScreenPtr pScreen)
 {

render/matrix.c

@@ -84,7 +84,6 @@ _X_EXPORT Bool
 PictureTransformPoint (PictTransformPtr transform,
 		       PictVectorPtr	vector)
 {
-    PictVector	    result;
     int		    i, j;
     xFixed_32_32    partial;
     xFixed_34_30    v[3];
@@ -227,7 +226,7 @@ PictureTransformTranslate (PictTransformPtr forward,
     return TRUE;
 }
 
-_X_EXPORT void
+_X_EXPORT Bool
 PictureTransformBounds (BoxPtr b, PictTransformPtr matrix)
 {
     PictVector	v[4];
@@ -240,7 +239,8 @@ PictureTransformBounds (BoxPtr b, PictTransformPtr matrix)
     v[3].vector[0] = F (b->x1);    v[3].vector[1] = F (b->y2);	v[3].vector[2] = F(1);
     for (i = 0; i < 4; i++)
     {
-	PictureTransformPoint (matrix, &v[i]);
+	if (!PictureTransformPoint (matrix, &v[i]))
+	    return FALSE;
 	x1 = xFixedToInt (v[i].vector[0]);
 	y1 = xFixedToInt (v[i].vector[1]);
 	x2 = xFixedToInt (xFixedCeil (v[i].vector[0]));
@@ -258,6 +258,7 @@ PictureTransformBounds (BoxPtr b, PictTransformPtr matrix)
 	    if (y2 > b->y2) b->y2 = y2;
 	}
     }
+    return TRUE;
 }
 
 _X_EXPORT Bool

render/picturestr.h

@@ -724,7 +724,7 @@ PictureTransformTranslate (PictTransformPtr forward,
 			   PictTransformPtr reverse,
 			   xFixed tx, xFixed ty);
 
-void
+Bool
 PictureTransformBounds (BoxPtr b, PictTransformPtr matrix);
 
 Bool