simple-damage: fix rotating-transform option

Previously !51 (closed), rebased and different branch.

If --rotating-transform is set, the damage calculation of the previous position of the ball has to be done with the old transform value, not the already updated one.

Note: this is still broken when using in combination with surface damage and viewport, as it's a rather complicated scenario and surface-damage is kinda legacy anyways.

This helps a lot on weston but there seem to be some some issues left in the compositor. On mutter it renders perfectly, with or without --use-damage-buffer.

Helps with: #155

Context: !41 (closed) (comment 54445)

clients/simple-damage.c

@@ -85,6 +85,7 @@ struct window {
 	enum window_flags flags;
 	int scale;
 	enum wl_output_transform transform;
+	enum wl_output_transform previous_transform;
 
 	struct {
 		float x, y; /* position in pixels */
@@ -447,14 +448,16 @@ paint_circle(uint32_t *pixels, int pitch, float x, float y, int radius,
 }
 
 static void
-window_get_transformed_ball(struct window *window, float *bx, float *by)
+window_get_transformed_ball(struct window *window,
+			     enum wl_output_transform transform,
+			     float *bx, float *by)
 {
 	float wx, wy;
 
 	wx = window->ball.x;
 	wy = window->ball.y;
 
-	switch (window->transform) {
+	switch (transform) {
 	default:
 	case WL_OUTPUT_TRANSFORM_NORMAL:
 		*bx = wx;
@@ -502,6 +505,111 @@ window_get_transformed_ball(struct window *window, float *bx, float *by)
 
 static const struct wl_callback_listener frame_listener;
 
+static void
+get_offset(struct window *window, enum wl_output_transform transform,
+	    int bwidth, int bheight, int *off_x, int *off_y) {
+
+	/* Offset the drawing region */
+	int tx = (window->width / 3) * window->scale;
+	int ty = (window->height / 5) * window->scale;
+	switch (transform) {
+	default:
+	case WL_OUTPUT_TRANSFORM_NORMAL:
+		*off_y = ty;
+		*off_x = tx;
+		break;
+	case WL_OUTPUT_TRANSFORM_90:
+		*off_y = tx;
+		*off_x = bwidth - ty;
+		break;
+	case WL_OUTPUT_TRANSFORM_180:
+		*off_y = bheight - ty;
+		*off_x = bwidth - tx;
+		break;
+	case WL_OUTPUT_TRANSFORM_270:
+		*off_y = bheight - tx;
+		*off_x = ty;
+		break;
+	case WL_OUTPUT_TRANSFORM_FLIPPED:
+		*off_y = ty;
+		*off_x = bwidth - tx;
+		break;
+	case WL_OUTPUT_TRANSFORM_FLIPPED_90:
+		*off_y = bheight - tx;
+		*off_x = bwidth - ty;
+		break;
+	case WL_OUTPUT_TRANSFORM_FLIPPED_180:
+		*off_y = bheight - ty;
+		*off_x = tx;
+		break;
+	case WL_OUTPUT_TRANSFORM_FLIPPED_270:
+		*off_y = tx;
+		*off_x = ty;
+		break;
+	}
+}
+
+static void
+damage_old_ball_position(struct window *window, int bradius, int bwidth,
+			  int bheight)
+{
+	float bx, by;
+	int off_x = 0, off_y = 0;
+
+	if (window->flags & WINDOW_FLAG_USE_DAMAGE_BUFFER) {
+		if (window->flags & WINDOW_FLAG_ROTATING_TRANSFORM) {
+			window_get_transformed_ball(window,
+				window->previous_transform, &bx, &by);
+			if (window->viewport)
+				get_offset (window, window->previous_transform,
+					bwidth, bheight, &off_x, &off_y);
+		}
+		else {
+			window_get_transformed_ball(window, window->transform,
+				&bx, &by);
+			if (window->viewport)
+				get_offset (window, window->transform, bwidth,
+					bheight, &off_x, &off_y);
+		}
+
+		wl_surface_damage_buffer(window->surface,
+					 bx - bradius + off_x,
+					 by - bradius + off_y,
+					 bradius * 2 + 1,
+					 bradius * 2 + 1);
+	} else {
+		if (window->flags & WINDOW_FLAG_ROTATING_TRANSFORM) {
+			switch (window->previous_transform) {
+				default:
+				case WL_OUTPUT_TRANSFORM_NORMAL:
+				case WL_OUTPUT_TRANSFORM_90:
+				case WL_OUTPUT_TRANSFORM_FLIPPED:
+				case WL_OUTPUT_TRANSFORM_FLIPPED_90:
+					bx = window->width - window->ball.x;
+					by = window->height - window->ball.y;
+					break;
+				case WL_OUTPUT_TRANSFORM_180:
+				case WL_OUTPUT_TRANSFORM_270:
+				case WL_OUTPUT_TRANSFORM_FLIPPED_180:
+				case WL_OUTPUT_TRANSFORM_FLIPPED_270:
+					bx = window->ball.x;
+					by = window->height - window->ball.y;
+					break;
+			}
+		}
+		else {
+			bx = window->ball.x;
+			by = window->ball.y;
+		}
+
+		wl_surface_damage(window->surface,
+				  bx - window->ball.radius,
+				  by - window->ball.radius,
+				  window->ball.radius * 2 + 1,
+				  window->ball.radius * 2 + 1);
+	}
+}
+
 static void
 redraw(void *data, struct wl_callback *callback, uint32_t time)
 {
@@ -521,8 +629,10 @@ redraw(void *data, struct wl_callback *callback, uint32_t time)
 
 	/* Rotate the damage, but keep the even/odd parity so the
 	 * dimensions of the buffers don't change */
-	if (window->flags & WINDOW_FLAG_ROTATING_TRANSFORM)
+	if (window->flags & WINDOW_FLAG_ROTATING_TRANSFORM) {
+		window->previous_transform = window->transform;
 		window->transform = (window->transform + 2) % 8;
+	}
 
 	switch (window->transform) {
 	default:
@@ -548,7 +658,6 @@ redraw(void *data, struct wl_callback *callback, uint32_t time)
 	bradius = window->ball.radius * window->scale;
 
 	if (window->viewport) {
-		int tx, ty;
 		/* Fill the whole thing with red to detect viewport errors */
 		paint_box(buffer->shm_data, bpitch, 0, 0, bwidth, bheight,
 			  0xffff0000);
@@ -561,43 +670,9 @@ redraw(void *data, struct wl_callback *callback, uint32_t time)
 		bheight /= 2;
 
 		/* Offset the drawing region */
-		tx = (window->width / 3) * window->scale;
-		ty = (window->height / 5) * window->scale;
-		switch (window->transform) {
-		default:
-		case WL_OUTPUT_TRANSFORM_NORMAL:
-			off_y = ty;
-			off_x = tx;
-			break;
-		case WL_OUTPUT_TRANSFORM_90:
-			off_y = tx;
-			off_x = bwidth - ty;
-			break;
-		case WL_OUTPUT_TRANSFORM_180:
-			off_y = bheight - ty;
-			off_x = bwidth - tx;
-			break;
-		case WL_OUTPUT_TRANSFORM_270:
-			off_y = bheight - tx;
-			off_x = ty;
-			break;
-		case WL_OUTPUT_TRANSFORM_FLIPPED:
-			off_y = ty;
-			off_x = bwidth - tx;
-			break;
-		case WL_OUTPUT_TRANSFORM_FLIPPED_90:
-			off_y = bheight - tx;
-			off_x = bwidth - ty;
-			break;
-		case WL_OUTPUT_TRANSFORM_FLIPPED_180:
-			off_y = bheight - ty;
-			off_x = tx;
-			break;
-		case WL_OUTPUT_TRANSFORM_FLIPPED_270:
-			off_y = tx;
-			off_x = ty;
-			break;
-		}
+		get_offset(window, window->transform, bwidth, bheight, &off_x, &off_y);
+
+
 		wp_viewport_set_source(window->viewport,
 				       wl_fixed_from_int(window->width / 3),
 				       wl_fixed_from_int(window->height / 5),
@@ -619,24 +694,11 @@ redraw(void *data, struct wl_callback *callback, uint32_t time)
 	paint_box(buffer->shm_data, bpitch, off_x + bborder, off_y + bborder,
 		  bwidth - 2 * bborder, bheight - 2 * bborder, 0x80000000);
 
-	/* Damage where the ball was */
-	if (window->flags & WINDOW_FLAG_USE_DAMAGE_BUFFER) {
-		window_get_transformed_ball(window, &bx, &by);
-		wl_surface_damage_buffer(window->surface,
-					 bx - bradius + off_x,
-					 by - bradius + off_y,
-					 bradius * 2 + 1,
-					 bradius * 2 + 1);
-	} else {
-		wl_surface_damage(window->surface,
-				  window->ball.x - window->ball.radius,
-				  window->ball.y - window->ball.radius,
-				  window->ball.radius * 2 + 1,
-				  window->ball.radius * 2 + 1);
-	}
+	damage_old_ball_position(window, bradius, bwidth, bheight);
+
 	window_advance_game(window, time);
 
-	window_get_transformed_ball(window, &bx, &by);
+	window_get_transformed_ball(window, window->transform, &bx, &by);
 
 	/* Paint the ball */
 	paint_circle(buffer->shm_data, bpitch, off_x + bx, off_y + by,