glx: Additional paranoia in __glXGetAnswerBuffer / __GLX_GET_ANSWER_BUFFER (v2) [CVE-2014-8093 3/6]

If the computed reply size is negative, something went wrong, treat it
as an error.

v2: Be more careful about size_t being unsigned (Matthieu Herrb)
v3: SIZE_MAX not SIZE_T_MAX (Alan Coopersmith)

Reviewed-by: Julien Cristau <jcristau@debian.org>
Reviewed-by: Michal Srb <msrb@suse.com>
Reviewed-by: Andy Ritger <aritger@nvidia.com>
Signed-off-by: Adam Jackson <ajax@redhat.com>
Signed-off-by: Alan Coopersmith <alan.coopersmith@oracle.com>

glx/indirect_util.c

@@ -76,9 +76,14 @@ __glXGetAnswerBuffer(__GLXclientState * cl, size_t required_size,
     const unsigned mask = alignment - 1;
 
     if (local_size < required_size) {
-        const size_t worst_case_size = required_size + alignment;
+        size_t worst_case_size;
         intptr_t temp_buf;
 
+        if (required_size < SIZE_MAX - alignment)
+            worst_case_size = required_size + alignment;
+        else
+            return NULL;
+
         if (cl->returnBufSize < worst_case_size) {
             void *temp = realloc(cl->returnBuf, worst_case_size);
 

glx/unpack.h

@@ -83,7 +83,8 @@ extern xGLXSingleReply __glXReply;
 ** pointer.
 */
 #define __GLX_GET_ANSWER_BUFFER(res,cl,size,align)			 \
-    if ((size) > sizeof(answerBuffer)) {				 \
+    if (size < 0) return BadLength;                                      \
+    else if ((size) > sizeof(answerBuffer)) {				 \
 	int bump;							 \
 	if ((cl)->returnBufSize < (size)+(align)) {			 \
 	    (cl)->returnBuf = (GLbyte*)realloc((cl)->returnBuf,	 	 \