Fixes SIGFPE, closes #6196.
Unified · Split
| @@ -63,7 +63,7 @@ typedef struct SCPRContext { · 62 unmodified lines | |||
| 63 | int cxshift; | 63 | int cxshift; |
| 64 | 64 | ||
| 65 | int (*get_freq)(RangeCoder *rc, unsigned total_freq, unsigned *freq); | 65 | int (*get_freq)(RangeCoder *rc, unsigned total_freq, unsigned *freq); |
| 66 | void (*decode)(GetByteContext *gb, RangeCoder *rc, unsigned cumFreq, unsigned freq, unsigned total_freq); | 66 | int (*decode)(GetByteContext *gb, RangeCoder *rc, unsigned cumFreq, unsigned freq, unsigned total_freq); |
| 67 | } SCPRContext; | 67 | } SCPRContext; |
| 68 | 68 | ||
| 69 | static void init_rangecoder(RangeCoder *rc, GetByteContext *gb) | 69 | static void init_rangecoder(RangeCoder *rc, GetByteContext *gb) |
| @@ -130,7 +130,7 @@ static void reinit_tables(SCPRContext *s) · 60 unmodified lines | |||
| 130 | s->mv_model[1][512] = 512; | 130 | s->mv_model[1][512] = 512; |
| 131 | } | 131 | } |
| 132 | 132 | ||
| 133 | static void decode(GetByteContext *gb, RangeCoder *rc, unsigned cumFreq, unsigned freq, unsigned total_freq) | 133 | static int decode(GetByteContext *gb, RangeCoder *rc, unsigned cumFreq, unsigned freq, unsigned total_freq) |
| 134 | { | 134 | { |
| 135 | rc->code -= cumFreq * rc->range; | 135 | rc->code -= cumFreq * rc->range; |
| 136 | rc->range *= freq; | 136 | rc->range *= freq; |
| @@ -140,6 +140,8 @@ static void decode(GetByteContext *gb, RangeCoder *rc, unsigned cumFreq, unsigne · 3 unmodified lines | |||
| 140 | rc->code = (rc->code << 8) | byte; | 140 | rc->code = (rc->code << 8) | byte; |
| 141 | rc->range <<= 8; | 141 | rc->range <<= 8; |
| 142 | } | 142 | } |
| 143 | |||
| 144 | return 0; | ||
| 143 | } | 145 | } |
| 144 | 146 | ||
| 145 | static int get_freq(RangeCoder *rc, unsigned total_freq, unsigned *freq) | 147 | static int get_freq(RangeCoder *rc, unsigned total_freq, unsigned *freq) |
| @@ -157,9 +159,14 @@ static int get_freq(RangeCoder *rc, unsigned total_freq, unsigned *freq) · 11 unmodified lines | |||
| 157 | return 0; | 159 | return 0; |
| 158 | } | 160 | } |
| 159 | 161 | ||
| 160 | static void decode0(GetByteContext *gb, RangeCoder *rc, unsigned cumFreq, unsigned freq, unsigned total_freq) | 162 | static int decode0(GetByteContext *gb, RangeCoder *rc, unsigned cumFreq, unsigned freq, unsigned total_freq) |
| 161 | { | 163 | { |
| 162 | int t = rc->range * (uint64_t)cumFreq / total_freq; | 164 | int t; |
| 165 | |||
| 166 | if (total_freq == 0) | ||
| 167 | return AVERROR_INVALIDDATA; | ||
| 168 | |||
| 169 | t = rc->range * (uint64_t)cumFreq / total_freq; | ||
| 163 | 170 | ||
| 164 | rc->code1 += t + 1; | 171 | rc->code1 += t + 1; |
| 165 | rc->range = rc->range * (uint64_t)(freq + cumFreq) / total_freq - (t + 1); | 172 | rc->range = rc->range * (uint64_t)(freq + cumFreq) / total_freq - (t + 1); |
| @@ -170,6 +177,8 @@ static void decode0(GetByteContext *gb, RangeCoder *rc, unsigned cumFreq, unsign · 4 unmodified lines | |||
| 170 | rc->code1 <<= 8; | 177 | rc->code1 <<= 8; |
| 171 | rc->range <<= 8; | 178 | rc->range <<= 8; |
| 172 | } | 179 | } |
| 180 | |||
| 181 | return 0; | ||
| 173 | } | 182 | } |
| 174 | 183 | ||
| 175 | static int get_freq0(RangeCoder *rc, unsigned total_freq, unsigned *freq) | 184 | static int get_freq0(RangeCoder *rc, unsigned total_freq, unsigned *freq) |
| @@ -202,7 +211,8 @@ static int decode_value(SCPRContext *s, unsigned *cnt, unsigned maxc, unsigned s · 26 unmodified lines | |||
| 202 | break; | 211 | break; |
| 203 | c++; | 212 | c++; |
| 204 | } | 213 | } |
| 205 | s->decode(gb, rc, cumfr, cnt_c, totfr); | 214 | if ((ret = s->decode(gb, rc, cumfr, cnt_c, totfr)) < 0) |
| 215 | return ret; | ||
| 206 | 216 | ||
| 207 | cnt[c] = cnt_c + step; | 217 | cnt[c] = cnt_c + step; |
| 208 | totfr += step; | 218 | totfr += step; |
| @@ -251,7 +261,10 @@ static int decode_unit(SCPRContext *s, PixelModel *pixel, unsigned step, unsigne · 42 unmodified lines | |||
| 251 | break; | 261 | break; |
| 252 | c++; | 262 | c++; |
| 253 | } | 263 | } |
| 254 | s->decode(gb, rc, cumfr, cnt_c, totfr); | 264 | |
| 265 | if ((ret = s->decode(gb, rc, cumfr, cnt_c, totfr)) < 0) | ||
| 266 | return ret; | ||
| 267 | |||
| 255 | pixel->freq[c] = cnt_c + step; | 268 | pixel->freq[c] = cnt_c + step; |
| 256 | pixel->lookup[x] = cnt_x + step; | 269 | pixel->lookup[x] = cnt_x + step; |
| 257 | totfr += step; | 270 | totfr += step; |