Skip to content

r300: fix UB caused by 1 << 31 and 2 << 30

Filip Gawin requested to merge gawin/mesa:r300/ub_bitshifts into main

There are also two cases for which I don't have idea:

compiler.RequiredOutputs = ~(~0 << (vs->info.num_outputs + 1));
../src/gallium/drivers/r300/r300_vs.c:252:37: runtime error: left shift of negative value -1
    #0 0x7f2c934f55f8 in r300_translate_vertex_shader ../src/gallium/drivers/r300/r300_vs.c:252
    #1 0x7f2c934c917d in r300_create_vs_state ../src/gallium/drivers/r300/r300_state.c:1921
    #2 0x7f2c912259fd in st_create_nir_shader ../src/mesa/state_tracker/st_program.c:540
    #3 0x7f2c91226cdc in st_create_common_variant ../src/mesa/state_tracker/st_program.c:854
    #4 0x7f2c9122f0df in st_get_common_variant ../src/mesa/state_tracker/st_program.c:986
    #5 0x7f2c9123623c in st_precompile_shader_variant ../src/mesa/state_tracker/st_program.c:1982
    #6 0x7f2c9123623c in st_finalize_program ../src/mesa/state_tracker/st_program.c:2042
    #7 0x7f2c912a1afb in st_program_string_notify ../src/mesa/state_tracker/st_cb_program.c:132
    #8 0x7f2c9211defa in _mesa_get_fixed_func_vertex_program ../src/mesa/main/ffvertex_prog.c:1726
    #9 0x7f2c91a1db66 in update_program ../src/mesa/main/state.c:283
    #10 0x7f2c91a20128 in _mesa_update_state_locked ../src/mesa/main/state.c:468
    #11 0x7f2c91a21010 in _mesa_update_state ../src/mesa/main/state.c:501
    #12 0x7f2c918cbbb7 in check_extra ../src/mesa/main/get.c:1439
    #13 0x7f2c918cbbb7 in find_value ../src/mesa/main/get.c:1611
    #14 0x7f2c918d7841 in _mesa_GetIntegerv ../src/mesa/main/get.c:1979
    #15 0x562581b7b2a5  (/usr/bin/xonotic-sdl+0x2812a5)
    #16 0x562581a2d3b2  (/usr/bin/xonotic-sdl+0x1333b2)
    #17 0x562581948b65  (/usr/bin/xonotic-sdl+0x4eb65)
    #18 0x562581b7a256  (/usr/bin/xonotic-sdl+0x280256)
    #19 0x562581b7d3e4  (/usr/bin/xonotic-sdl+0x2833e4)
    #20 0x562581a5a2b1  (/usr/bin/xonotic-sdl+0x1602b1)
    #21 0x5625819ac2ae  (/usr/bin/xonotic-sdl+0xb22ae)
    #22 0x562581a57db4  (/usr/bin/xonotic-sdl+0x15ddb4)
    #23 0x562581a59214  (/usr/bin/xonotic-sdl+0x15f214)
    #24 0x5625819450fb  (/usr/bin/xonotic-sdl+0x4b0fb)
    #25 0x7f2c97c5bb24 in __libc_start_main (/usr/lib/libc.so.6+0x27b24)
    #26 0x56258194515d  (/usr/bin/xonotic-sdl+0x4b15d)

and second:

vstream->vap_prog_stream_cntl_ext[i >> 1] |= swizzle << 16;

(swizzle is uint16_t)

../src/gallium/drivers/r300/r300_state.c:1824:66: runtime error: left shift of 63112 by 16 places cannot be represented in type 'int'
    #0 0x7f2c90f88e6f in r300_vertex_psc ../src/gallium/drivers/r300/r300_state.c:1824
    #1 0x7f2c90f88e6f in r300_create_vertex_elements_state ../src/gallium/drivers/r300/r300_state.c:1870
    #2 0x7f2c936a1d51 in util_blitter_create ../src/gallium/auxiliary/util/u_blitter.c:318
    #3 0x7f2c934877be in r300_create_context ../src/gallium/drivers/r300/r300_context.c:434
    #4 0x7f2c91197873 in st_api_create_context ../src/mesa/state_tracker/st_manager.c:935
    #5 0x7f2c90fad0da in dri_create_context ../src/gallium/frontends/dri/dri_context.c:163
    #6 0x7f2c92fce06c in driCreateContextAttribs ../src/mesa/drivers/dri/common/dri_util.c:480
    #7 0x7f2c961209d3 in dri3_create_context_attribs ../src/glx/dri3_glx.c:310
    #8 0x7f2c960aee2c in dri_common_create_context ../src/glx/dri_common.c:605
    #9 0x7f2c960c1784 in CreateContext ../src/glx/glxcmds.c:347
    #10 0x7f2c960c2c2e in glXCreateContext ../src/glx/glxcmds.c:472
    #11 0x7f2c98051474  (/usr/lib/libSDL2-2.0.so.0+0xed474)
    #12 0x7f2c9801a935  (/usr/lib/libSDL2-2.0.so.0+0xb6935)
    #13 0x7f2c9801d081  (/usr/lib/libSDL2-2.0.so.0+0xb9081)
    #14 0x7f2c9801da2c  (/usr/lib/libSDL2-2.0.so.0+0xb9a2c)
    #15 0x7f2c97f85317  (/usr/lib/libSDL2-2.0.so.0+0x21317)
    #16 0x56258194785e  (/usr/bin/xonotic-sdl+0x4d85e)
    #17 0x562581a57d20  (/usr/bin/xonotic-sdl+0x15dd20)
    #18 0x562581a59214  (/usr/bin/xonotic-sdl+0x15f214)
    #19 0x5625819450fb  (/usr/bin/xonotic-sdl+0x4b0fb)
    #20 0x7f2c97c5bb24 in __libc_start_main (/usr/lib/libc.so.6+0x27b24)
    #21 0x56258194515d  (/usr/bin/xonotic-sdl+0x4b15d)

@mareko

Merge request reports