Currently, packed16togbra16() always sets the alpha value to 0xFFFF, without taking the bit depth into consideration. This causes a bug on x86, which can be reproduced with: ./libswscale/tests/swscale -unscaled 1 -src xyz12le -dst gbrap12be The problem arises in ff_hscale14to15_4_ssse3(), in the conversion from gbrap12be to yuva444p, which comes after the conversion from xyz12le to gbrap12be. It has something to do with pmaddwd not working on unsigned values. There is some code to deal with 0xFFFF if the input has a bit depth of 16, but not for bit depths < 16. We could fix ff_hscale14to15_4_ssse3() to also work correctly with 0xFFFF on bit depths < 16, or we could just not write 0xFFFF there in the first place, which is what this commit does.
Unified · Split
| @@ -699,7 +699,7 @@ static void packed16togbra16(const uint8_t *src, int srcStride, · 698 unmodified lines | ||
| 699 | 699 | dst[0][x] = av_bswap16(av_bswap16(*src_line++) >> shift); |
| 700 | 700 | dst[1][x] = av_bswap16(av_bswap16(*src_line++) >> shift); |
| 701 | 701 | dst[2][x] = av_bswap16(av_bswap16(*src_line++) >> shift); |
| 702 | − dst[3][x] = 0xFFFF; | |
| 702 | + dst[3][x] = av_bswap16(0xFFFF >> shift); | |
| 703 | 703 | } |
| 704 | 704 | } else if (src_alpha) { |
| 705 | 705 | for (x = 0; x < width; x++) { |
| @@ -729,7 +729,7 @@ static void packed16togbra16(const uint8_t *src, int srcStride, · 23 unmodified lines | ||
| 729 | 729 | dst[0][x] = av_bswap16(*src_line++ >> shift); |
| 730 | 730 | dst[1][x] = av_bswap16(*src_line++ >> shift); |
| 731 | 731 | dst[2][x] = av_bswap16(*src_line++ >> shift); |
| 732 | − dst[3][x] = 0xFFFF; | |
| 732 | + dst[3][x] = av_bswap16(0xFFFF >> shift); | |
| 733 | 733 | } |
| 734 | 734 | } else if (src_alpha) { |
| 735 | 735 | for (x = 0; x < width; x++) { |
| @@ -759,7 +759,7 @@ static void packed16togbra16(const uint8_t *src, int srcStride, · 23 unmodified lines | ||
| 759 | 759 | dst[0][x] = av_bswap16(*src_line++) >> shift; |
| 760 | 760 | dst[1][x] = av_bswap16(*src_line++) >> shift; |
| 761 | 761 | dst[2][x] = av_bswap16(*src_line++) >> shift; |
| 762 | − dst[3][x] = 0xFFFF; | |
| 762 | + dst[3][x] = 0xFFFF >> shift; | |
| 763 | 763 | } |
| 764 | 764 | } else if (src_alpha) { |
| 765 | 765 | for (x = 0; x < width; x++) { |
| @@ -789,7 +789,7 @@ static void packed16togbra16(const uint8_t *src, int srcStride, · 23 unmodified lines | ||
| 789 | 789 | dst[0][x] = *src_line++ >> shift; |
| 790 | 790 | dst[1][x] = *src_line++ >> shift; |
| 791 | 791 | dst[2][x] = *src_line++ >> shift; |
| 792 | − dst[3][x] = 0xFFFF; | |
| 792 | + dst[3][x] = 0xFFFF >> shift; | |
| 793 | 793 | } |
| 794 | 794 | } else if (src_alpha) { |
| 795 | 795 | for (x = 0; x < width; x++) { |