JBIG2: avoid abort() on large memory allocation

Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=24772

When numInputSyms + numNewSyms is large enough, a fatal out of memory
allocation can occur in JArithmeticDecoderStats() constructor per

```
    #0 0xf7f6bf19 in [vdso]
    #1 0xf7d40d08 in gsignal (/lib32/libc.so.6+0x2bd08)
    #2 0xf7d42206 in abort (/lib32/libc.so.6+0x2d206)
    #3 0xbdc0049 in gmalloc(unsigned int, bool) gdal/poppler/goo/gmem.h:52:5
    #4 0xbdf3c61 in gmallocn(int, int, bool) gdal/poppler/goo/gmem.h:119:12
    #5 0xc1391fd in JArithmeticDecoderStats::JArithmeticDecoderStats(int) gdal/poppler/poppler/JArithmeticDecoder.cc:36:30
    #6 0xc1130d5 in JBIG2Stream::resetIntStats(int) gdal/poppler/poppler/JBIG2Stream.cc:4052:25
    #7 0xc1083df in JBIG2Stream::readSymbolDictSeg(unsigned int, unsigned int, unsigned int*, unsigned int) gdal/poppler/poppler/JBIG2Stream.cc:1624:9
    #8 0xc105305 in JBIG2Stream::readSegments() gdal/poppler/poppler/JBIG2Stream.cc:1318:18
    #9 0xc103f5a in JBIG2Stream::reset() gdal/poppler/poppler/JBIG2Stream.cc:1142:5
```

Avoid it and return nicely.
parent c063a74d
......@@ -33,8 +33,10 @@
JArithmeticDecoderStats::JArithmeticDecoderStats(int contextSizeA)
{
contextSize = contextSizeA;
cxTab = (unsigned char *)gmallocn(contextSize, sizeof(unsigned char));
reset();
cxTab = (unsigned char *)gmallocn_checkoverflow(contextSize, sizeof(unsigned char));
if (cxTab) {
reset();
}
}
JArithmeticDecoderStats::~JArithmeticDecoderStats()
......
......@@ -44,6 +44,7 @@ public:
int getContextSize() { return contextSize; }
void copyFrom(JArithmeticDecoderStats *stats);
void setEntry(unsigned int cx, int i, int mps);
bool isValid() const { return cxTab != nullptr; }
private:
unsigned char *cxTab; // cxTab[cx] = (i[cx] << 1) + mps[cx]
......
......@@ -1621,7 +1621,9 @@ bool JBIG2Stream::readSymbolDictSeg(unsigned int segNum, unsigned int length, un
} else {
resetGenericStats(sdTemplate, nullptr);
}
resetIntStats(symCodeLen);
if (!resetIntStats(symCodeLen)) {
goto syntaxError;
}
arithDecoder->start();
}
......@@ -1716,6 +1718,9 @@ bool JBIG2Stream::readSymbolDictSeg(unsigned int segNum, unsigned int length, un
huffDecoder->reset();
arithDecoder->start();
} else {
if (iaidStats == nullptr) {
goto syntaxError;
}
symID = arithDecoder->decodeIAID(symCodeLen, iaidStats);
arithDecoder->decodeInt(&refDX, iardxStats);
arithDecoder->decodeInt(&refDY, iardyStats);
......@@ -2127,7 +2132,9 @@ void JBIG2Stream::readTextRegionSeg(unsigned int segNum, bool imm, bool lossless
}
if (!huff) {
resetIntStats(symCodeLen);
if (!resetIntStats(symCodeLen)) {
return;
}
arithDecoder->start();
}
if (refine) {
......@@ -2252,6 +2259,10 @@ JBIG2Bitmap *JBIG2Stream::readTextRegion(bool huff, bool refine, int w, int h, u
symID = huffDecoder->readBits(symCodeLen);
}
} else {
if (iaidStats == nullptr) {
delete bitmap;
return nullptr;
}
symID = arithDecoder->decodeIAID(symCodeLen, iaidStats);
}
......@@ -4030,7 +4041,7 @@ void JBIG2Stream::resetRefinementStats(unsigned int templ, JArithmeticDecoderSta
}
}
void JBIG2Stream::resetIntStats(int symCodeLen)
bool JBIG2Stream::resetIntStats(int symCodeLen)
{
iadhStats->reset();
iadwStats->reset();
......@@ -4045,12 +4056,18 @@ void JBIG2Stream::resetIntStats(int symCodeLen)
iardwStats->reset();
iardhStats->reset();
iariStats->reset();
if (iaidStats->getContextSize() == 1 << (symCodeLen + 1)) {
if (iaidStats != nullptr && iaidStats->getContextSize() == 1 << (symCodeLen + 1)) {
iaidStats->reset();
} else {
delete iaidStats;
iaidStats = new JArithmeticDecoderStats(1 << (symCodeLen + 1));
if (!iaidStats->isValid()) {
delete iaidStats;
iaidStats = nullptr;
return false;
}
}
return true;
}
bool JBIG2Stream::readUByte(unsigned int *x)
......
......@@ -88,7 +88,7 @@ private:
void discardSegment(unsigned int segNum);
void resetGenericStats(unsigned int templ, JArithmeticDecoderStats *prevStats);
void resetRefinementStats(unsigned int templ, JArithmeticDecoderStats *prevStats);
void resetIntStats(int symCodeLen);
bool resetIntStats(int symCodeLen);
bool readUByte(unsigned int *x);
bool readByte(int *x);
bool readUWord(unsigned int *x);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment