History

avcodec/mpegvideo_enc: use 64bit multiplication in dct_quantize_trellis_c and dct_quantize_c

Fixes corruption with: ffmpeg -t 1 -filter_complex "sine=f=21,showwaves=scale=cbrt:mode=line:colors=white:draw=full" -c:v mpeg2video -non_linear_quant 1 -qmin 1 -qmax 1 -cpuflags 0 out.mpg or ffmpeg -t 1 -filter_complex "sine=f=21,showwaves=scale=cbrt:mode=line:colors=white:draw=full" -c:v mpeg2video -non_linear_quant 1 -qmin 1 -qmax 1 -trellis 1 out.mpg

Marton Balint committed 1 year ago

Commit ec4d3dc1 parent ed268121 file · +11 −11Browse files

Showing the changes to libavcodec (1 of 1 file). Show all

Unified · Split

libavcodec/mpegvideo_enc.c+11−11
@@ -3998,9 +3998,9 @@ static int dct_quantize_trellis_c(MpegEncContext *s, · 3997 unmodified lines
3998 3998
3999 for(i=63; i>=start_i; i--) {3999 for(i=63; i>=start_i; i--) {
4000 const int j = scantable[i];4000 const int j = scantable[i];
4001 int level = block[j] * qmat[j];4001 int64_t level = (int64_t)block[j] * qmat[j];
4002 4002
4003 if(((unsigned)(level+threshold1))>threshold2){4003 if(((uint64_t)(level+threshold1))>threshold2){
4004 last_non_zero = i;4004 last_non_zero = i;
4005 break;4005 break;
4006 }4006 }
@@ -4008,11 +4008,11 @@ static int dct_quantize_trellis_c(MpegEncContext *s, · 1 unmodified line
4008 4008
4009 for(i=start_i; i<=last_non_zero; i++) {4009 for(i=start_i; i<=last_non_zero; i++) {
4010 const int j = scantable[i];4010 const int j = scantable[i];
4011 int level = block[j] * qmat[j];4011 int64_t level = (int64_t)block[j] * qmat[j];
4012 4012
4013// if( bias+level >= (1<<(QMAT_SHIFT - 3))4013// if( bias+level >= (1<<(QMAT_SHIFT - 3))
4014// || bias-level >= (1<<(QMAT_SHIFT - 3))){4014// || bias-level >= (1<<(QMAT_SHIFT - 3))){
4015 if(((unsigned)(level+threshold1))>threshold2){4015 if(((uint64_t)(level+threshold1))>threshold2){
4016 if(level>0){4016 if(level>0){
4017 level= (bias + level)>>QMAT_SHIFT;4017 level= (bias + level)>>QMAT_SHIFT;
4018 coeff[0][i]= level;4018 coeff[0][i]= level;
@@ -4601,7 +4601,7 @@ static int dct_quantize_c(MpegEncContext *s, · 582 unmodified lines
4601 int16_t *block, int n,4601 int16_t *block, int n,
4602 int qscale, int *overflow)4602 int qscale, int *overflow)
4603{4603{
4604 int i, j, level, last_non_zero, q, start_i;4604 int i, last_non_zero, q, start_i;
4605 const int *qmat;4605 const int *qmat;
4606 const uint8_t *scantable;4606 const uint8_t *scantable;
4607 int bias;4607 int bias;
@@ -4641,10 +4641,10 @@ static int dct_quantize_c(MpegEncContext *s, · 33 unmodified lines
4641 threshold1= (1<<QMAT_SHIFT) - bias - 1;4641 threshold1= (1<<QMAT_SHIFT) - bias - 1;
4642 threshold2= (threshold1<<1);4642 threshold2= (threshold1<<1);
4643 for(i=63;i>=start_i;i--) {4643 for(i=63;i>=start_i;i--) {
4644 j = scantable[i];4644 const int j = scantable[i];
4645 level = block[j] * qmat[j];4645 int64_t level = (int64_t)block[j] * qmat[j];
4646 4646
4647 if(((unsigned)(level+threshold1))>threshold2){4647 if(((uint64_t)(level+threshold1))>threshold2){
4648 last_non_zero = i;4648 last_non_zero = i;
4649 break;4649 break;
4650 }else{4650 }else{
@@ -4652,12 +4652,12 @@ static int dct_quantize_c(MpegEncContext *s, · 1 unmodified line
4652 }4652 }
4653 }4653 }
4654 for(i=start_i; i<=last_non_zero; i++) {4654 for(i=start_i; i<=last_non_zero; i++) {
4655 j = scantable[i];4655 const int j = scantable[i];
4656 level = block[j] * qmat[j];4656 int64_t level = (int64_t)block[j] * qmat[j];
4657 4657
4658// if( bias+level >= (1<<QMAT_SHIFT)4658// if( bias+level >= (1<<QMAT_SHIFT)
4659// || bias-level >= (1<<QMAT_SHIFT)){4659// || bias-level >= (1<<QMAT_SHIFT)){
4660 if(((unsigned)(level+threshold1))>threshold2){4660 if(((uint64_t)(level+threshold1))>threshold2){
4661 if(level>0){4661 if(level>0){
4662 level= (bias + level)>>QMAT_SHIFT;4662 level= (bias + level)>>QMAT_SHIFT;
4663 block[j]= level;4663 block[j]= level;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t