avcodec/ffv1dec: compute end instead of hardcoding it and test for fltmap correctly

A step toward supporting remap for integer formats less than 16 bit

Michael Niedermayer committed 1 year ago

Commit 5ca14b21 parent 38135591 file · +2 −2Browse files

Unified · Split

libavcodec/ffv1dec.c+2−2

@@ -286,7 +286,7 @@ static int decode_current_mul(RangeCoder rc, uint8_t state[32], int mul, int m · 285 unmodified lines
286	286
287	287	static int decode_remap(FFV1Context f, FFV1SliceContext sc)
288	288	{
289		− unsigned int end = f->avctx->bits_per_raw_sample == 32 ? 0xFFFFFFFF : 0xFFFF;
	289	+ unsigned int end = (1LL<<f->avctx->bits_per_raw_sample) - 1;
290	290	int flip = sc->remap == 2 ? (end>>1) : 0;
291	291
292	292	for (int p= 0; p < 1 + 2*f->chroma_planes + f->transparency; p++) {
@@ -328,7 +328,7 @@ static int decode_remap(FFV1Context f, FFV1SliceContext sc) · 35 unmodified lines
328	328	break;
329	329	if (i - 1 > end \|\| j > 65535)
330	330	return AVERROR_INVALIDDATA;
331		− if (end == 0xFFFF) {
	331	+ if (end <= 0xFFFF) {
332	332	sc->fltmap [p][j++] = i ^ ((i& 0x8000) ? 0 : flip);
333	333	} else
334	334	sc->fltmap32[p][j++] = i ^ ((i&0x80000000) ? 0 : flip);