History

h264: do not use 422 functions for monochrome

Fixes invalid memory access.

Anton Khirnov committed 12 years ago

Commit a03a6421 parent 9eef9eb6 files · +30 −30Browse files

Unified · Split

6 files
  1. libavcodec/arm/h264dsp_init_arm.c +1 −1
  2. libavcodec/h264dsp.c +6 −6
  3. libavcodec/h264pred.c +5 −5
  4. libavcodec/ppc/h264dsp.c +1 −1
  5. libavcodec/x86/h264_intrapred_init.c +10 −10
  6. libavcodec/x86/h264dsp_init.c +7 −7
libavcodec/arm/h264dsp_init_arm.c+1−1
@@ -91,7 +91,7 @@ static av_cold void h264dsp_init_neon(H264DSPContext *c, const int bit_depth, · 90 unmodified lines
9191 c->h264_idct_dc_add = ff_h264_idct_dc_add_neon;
9292 c->h264_idct_add16 = ff_h264_idct_add16_neon;
9393 c->h264_idct_add16intra = ff_h264_idct_add16intra_neon;
94− if (chroma_format_idc == 1)
94+ if (chroma_format_idc <= 1)
9595 c->h264_idct_add8 = ff_h264_idct_add8_neon;
9696 c->h264_idct8_add = ff_h264_idct8_add_neon;
9797 c->h264_idct8_dc_add = ff_h264_idct8_dc_add_neon;
libavcodec/h264dsp.c+6−6
@@ -104,13 +104,13 @@ av_cold void ff_h264dsp_init(H264DSPContext *c, const int bit_depth, · 103 unmodified lines
104104 c->h264_idct8_dc_add= FUNC(ff_h264_idct8_dc_add, depth);\
105105 c->h264_idct_add16 = FUNC(ff_h264_idct_add16, depth);\
106106 c->h264_idct8_add4 = FUNC(ff_h264_idct8_add4, depth);\
107− if (chroma_format_idc == 1)\
107+ if (chroma_format_idc <= 1)\
108108 c->h264_idct_add8 = FUNC(ff_h264_idct_add8, depth);\
109109 else\
110110 c->h264_idct_add8 = FUNC(ff_h264_idct_add8_422, depth);\
111111 c->h264_idct_add16intra= FUNC(ff_h264_idct_add16intra, depth);\
112112 c->h264_luma_dc_dequant_idct= FUNC(ff_h264_luma_dc_dequant_idct, depth);\
113− if (chroma_format_idc == 1)\
113+ if (chroma_format_idc <= 1)\
114114 c->h264_chroma_dc_dequant_idct= FUNC(ff_h264_chroma_dc_dequant_idct, depth);\
115115 else\
116116 c->h264_chroma_dc_dequant_idct= FUNC(ff_h264_chroma422_dc_dequant_idct, depth);\
@@ -131,20 +131,20 @@ av_cold void ff_h264dsp_init(H264DSPContext *c, const int bit_depth, · 14 unmodified lines
131131 c->h264_h_loop_filter_luma_intra= FUNC(h264_h_loop_filter_luma_intra, depth);\
132132 c->h264_h_loop_filter_luma_mbaff_intra= FUNC(h264_h_loop_filter_luma_mbaff_intra, depth);\
133133 c->h264_v_loop_filter_chroma= FUNC(h264_v_loop_filter_chroma, depth);\
134− if (chroma_format_idc == 1)\
134+ if (chroma_format_idc <= 1)\
135135 c->h264_h_loop_filter_chroma= FUNC(h264_h_loop_filter_chroma, depth);\
136136 else\
137137 c->h264_h_loop_filter_chroma= FUNC(h264_h_loop_filter_chroma422, depth);\
138− if (chroma_format_idc == 1)\
138+ if (chroma_format_idc <= 1)\
139139 c->h264_h_loop_filter_chroma_mbaff= FUNC(h264_h_loop_filter_chroma_mbaff, depth);\
140140 else\
141141 c->h264_h_loop_filter_chroma_mbaff= FUNC(h264_h_loop_filter_chroma422_mbaff, depth);\
142142 c->h264_v_loop_filter_chroma_intra= FUNC(h264_v_loop_filter_chroma_intra, depth);\
143− if (chroma_format_idc == 1)\
143+ if (chroma_format_idc <= 1)\
144144 c->h264_h_loop_filter_chroma_intra= FUNC(h264_h_loop_filter_chroma_intra, depth);\
145145 else\
146146 c->h264_h_loop_filter_chroma_intra= FUNC(h264_h_loop_filter_chroma422_intra, depth);\
147− if (chroma_format_idc == 1)\
147+ if (chroma_format_idc <= 1)\
148148 c->h264_h_loop_filter_chroma_mbaff_intra= FUNC(h264_h_loop_filter_chroma_mbaff_intra, depth);\
149149 else\
150150 c->h264_h_loop_filter_chroma_mbaff_intra= FUNC(h264_h_loop_filter_chroma422_mbaff_intra, depth);\
libavcodec/h264pred.c+5−5
@@ -472,7 +472,7 @@ av_cold void ff_h264_pred_init(H264PredContext *h, int codec_id, · 471 unmodified lines
472472 h->pred8x8l[TOP_DC_PRED ]= FUNCC(pred8x8l_top_dc , depth);\
473473 h->pred8x8l[DC_128_PRED ]= FUNCC(pred8x8l_128_dc , depth);\
474474 \
475− if (chroma_format_idc == 1) {\
475+ if (chroma_format_idc <= 1) {\
476476 h->pred8x8[VERT_PRED8x8 ]= FUNCC(pred8x8_vertical , depth);\
477477 h->pred8x8[HOR_PRED8x8 ]= FUNCC(pred8x8_horizontal , depth);\
478478 } else {\
@@ -480,7 +480,7 @@ av_cold void ff_h264_pred_init(H264PredContext *h, int codec_id, · 1 unmodified line
480480 h->pred8x8[HOR_PRED8x8 ]= FUNCC(pred8x16_horizontal , depth);\
481481 }\
482482 if (codec_id != AV_CODEC_ID_VP8) {\
483− if (chroma_format_idc == 1) {\
483+ if (chroma_format_idc <= 1) {\
484484 h->pred8x8[PLANE_PRED8x8]= FUNCC(pred8x8_plane , depth);\
485485 } else {\
486486 h->pred8x8[PLANE_PRED8x8]= FUNCC(pred8x16_plane , depth);\
@@ -488,7 +488,7 @@ av_cold void ff_h264_pred_init(H264PredContext *h, int codec_id, · 1 unmodified line
488488 } else\
489489 h->pred8x8[PLANE_PRED8x8]= FUNCD(pred8x8_tm_vp8);\
490490 if(codec_id != AV_CODEC_ID_RV40 && codec_id != AV_CODEC_ID_VP8){\
491− if (chroma_format_idc == 1) {\
491+ if (chroma_format_idc <= 1) {\
492492 h->pred8x8[DC_PRED8x8 ]= FUNCC(pred8x8_dc , depth);\
493493 h->pred8x8[LEFT_DC_PRED8x8]= FUNCC(pred8x8_left_dc , depth);\
494494 h->pred8x8[TOP_DC_PRED8x8 ]= FUNCC(pred8x8_top_dc , depth);\
@@ -514,7 +514,7 @@ av_cold void ff_h264_pred_init(H264PredContext *h, int codec_id, · 19 unmodified lines
514514 h->pred8x8[DC_129_PRED8x8]= FUNCC(pred8x8_129_dc , depth);\
515515 }\
516516 }\
517− if (chroma_format_idc == 1) {\
517+ if (chroma_format_idc <= 1) {\
518518 h->pred8x8[DC_128_PRED8x8 ]= FUNCC(pred8x8_128_dc , depth);\
519519 } else {\
520520 h->pred8x8[DC_128_PRED8x8 ]= FUNCC(pred8x16_128_dc , depth);\
@@ -548,7 +548,7 @@ av_cold void ff_h264_pred_init(H264PredContext *h, int codec_id, · 27 unmodified lines
548548 h->pred4x4_add [ HOR_PRED ]= FUNCC(pred4x4_horizontal_add , depth);\
549549 h->pred8x8l_add [VERT_PRED ]= FUNCC(pred8x8l_vertical_add , depth);\
550550 h->pred8x8l_add [ HOR_PRED ]= FUNCC(pred8x8l_horizontal_add , depth);\
551− if (chroma_format_idc == 1) {\
551+ if (chroma_format_idc <= 1) {\
552552 h->pred8x8_add [VERT_PRED8x8]= FUNCC(pred8x8_vertical_add , depth);\
553553 h->pred8x8_add [ HOR_PRED8x8]= FUNCC(pred8x8_horizontal_add , depth);\
554554 } else {\
libavcodec/ppc/h264dsp.c+1−1
@@ -750,7 +750,7 @@ av_cold void ff_h264dsp_init_ppc(H264DSPContext *c, const int bit_depth, · 749 unmodified lines
750750
751751 if (bit_depth == 8) {
752752 c->h264_idct_add = h264_idct_add_altivec;
753− if (chroma_format_idc == 1)
753+ if (chroma_format_idc <= 1)
754754 c->h264_idct_add8 = h264_idct_add8_altivec;
755755 c->h264_idct_add16 = h264_idct_add16_altivec;
756756 c->h264_idct_add16intra = h264_idct_add16intra_altivec;
libavcodec/x86/h264_intrapred_init.c+10−10
@@ -191,7 +191,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, · 190 unmodified lines
191191 if (EXTERNAL_MMX(cpu_flags)) {
192192 h->pred16x16[VERT_PRED8x8 ] = ff_pred16x16_vertical_8_mmx;
193193 h->pred16x16[HOR_PRED8x8 ] = ff_pred16x16_horizontal_8_mmx;
194− if (chroma_format_idc == 1) {
194+ if (chroma_format_idc <= 1) {
195195 h->pred8x8 [VERT_PRED8x8 ] = ff_pred8x8_vertical_8_mmx;
196196 h->pred8x8 [HOR_PRED8x8 ] = ff_pred8x8_horizontal_8_mmx;
197197 }
@@ -200,7 +200,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, · 2 unmodified lines
200200 h->pred8x8 [PLANE_PRED8x8 ] = ff_pred8x8_tm_vp8_8_mmx;
201201 h->pred4x4 [TM_VP8_PRED ] = ff_pred4x4_tm_vp8_8_mmx;
202202 } else {
203− if (chroma_format_idc == 1)
203+ if (chroma_format_idc <= 1)
204204 h->pred8x8 [PLANE_PRED8x8] = ff_pred8x8_plane_8_mmx;
205205 if (codec_id == AV_CODEC_ID_SVQ3) {
206206 if (cpu_flags & AV_CPU_FLAG_CMOV)
@@ -216,7 +216,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, · 9 unmodified lines
216216 if (EXTERNAL_MMXEXT(cpu_flags)) {
217217 h->pred16x16[HOR_PRED8x8 ] = ff_pred16x16_horizontal_8_mmxext;
218218 h->pred16x16[DC_PRED8x8 ] = ff_pred16x16_dc_8_mmxext;
219− if (chroma_format_idc == 1)
219+ if (chroma_format_idc <= 1)
220220 h->pred8x8[HOR_PRED8x8 ] = ff_pred8x8_horizontal_8_mmxext;
221221 h->pred8x8l [TOP_DC_PRED ] = ff_pred8x8l_top_dc_8_mmxext;
222222 h->pred8x8l [DC_PRED ] = ff_pred8x8l_dc_8_mmxext;
@@ -241,7 +241,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, · 18 unmodified lines
241241 h->pred4x4 [HOR_UP_PRED ] = ff_pred4x4_horizontal_up_8_mmxext;
242242 }
243243 if (codec_id == AV_CODEC_ID_SVQ3 || codec_id == AV_CODEC_ID_H264) {
244− if (chroma_format_idc == 1) {
244+ if (chroma_format_idc <= 1) {
245245 h->pred8x8[TOP_DC_PRED8x8 ] = ff_pred8x8_top_dc_8_mmxext;
246246 h->pred8x8[DC_PRED8x8 ] = ff_pred8x8_dc_8_mmxext;
247247 }
@@ -253,7 +253,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, · 5 unmodified lines
253253 h->pred4x4 [TM_VP8_PRED ] = ff_pred4x4_tm_vp8_8_mmxext;
254254 h->pred4x4 [VERT_PRED ] = ff_pred4x4_vertical_vp8_8_mmxext;
255255 } else {
256− if (chroma_format_idc == 1)
256+ if (chroma_format_idc <= 1)
257257 h->pred8x8 [PLANE_PRED8x8] = ff_pred8x8_plane_8_mmxext;
258258 if (codec_id == AV_CODEC_ID_SVQ3) {
259259 h->pred16x16[PLANE_PRED8x8 ] = ff_pred16x16_plane_svq3_8_mmxext;
@@ -280,7 +280,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, · 20 unmodified lines
280280 h->pred16x16[PLANE_PRED8x8 ] = ff_pred16x16_tm_vp8_8_sse2;
281281 h->pred8x8 [PLANE_PRED8x8 ] = ff_pred8x8_tm_vp8_8_sse2;
282282 } else {
283− if (chroma_format_idc == 1)
283+ if (chroma_format_idc <= 1)
284284 h->pred8x8 [PLANE_PRED8x8] = ff_pred8x8_plane_8_sse2;
285285 if (codec_id == AV_CODEC_ID_SVQ3) {
286286 h->pred16x16[PLANE_PRED8x8] = ff_pred16x16_plane_svq3_8_sse2;
@@ -295,7 +295,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, · 8 unmodified lines
295295 if (EXTERNAL_SSSE3(cpu_flags)) {
296296 h->pred16x16[HOR_PRED8x8 ] = ff_pred16x16_horizontal_8_ssse3;
297297 h->pred16x16[DC_PRED8x8 ] = ff_pred16x16_dc_8_ssse3;
298− if (chroma_format_idc == 1)
298+ if (chroma_format_idc <= 1)
299299 h->pred8x8 [HOR_PRED8x8 ] = ff_pred8x8_horizontal_8_ssse3;
300300 h->pred8x8l [TOP_DC_PRED ] = ff_pred8x8l_top_dc_8_ssse3;
301301 h->pred8x8l [DC_PRED ] = ff_pred8x8l_dc_8_ssse3;
@@ -311,7 +311,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, · 9 unmodified lines
311311 h->pred8x8 [PLANE_PRED8x8 ] = ff_pred8x8_tm_vp8_8_ssse3;
312312 h->pred4x4 [TM_VP8_PRED ] = ff_pred4x4_tm_vp8_8_ssse3;
313313 } else {
314− if (chroma_format_idc == 1)
314+ if (chroma_format_idc <= 1)
315315 h->pred8x8 [PLANE_PRED8x8] = ff_pred8x8_plane_8_ssse3;
316316 if (codec_id == AV_CODEC_ID_SVQ3) {
317317 h->pred16x16[PLANE_PRED8x8] = ff_pred16x16_plane_svq3_8_ssse3;
@@ -327,7 +327,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, · 9 unmodified lines
327327 h->pred4x4[DC_PRED ] = ff_pred4x4_dc_10_mmxext;
328328 h->pred4x4[HOR_UP_PRED ] = ff_pred4x4_horizontal_up_10_mmxext;
329329
330− if (chroma_format_idc == 1)
330+ if (chroma_format_idc <= 1)
331331 h->pred8x8[DC_PRED8x8 ] = ff_pred8x8_dc_10_mmxext;
332332
333333 h->pred8x8l[DC_128_PRED ] = ff_pred8x8l_128_dc_10_mmxext;
@@ -346,7 +346,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, · 12 unmodified lines
346346 h->pred4x4[VERT_RIGHT_PRED ] = ff_pred4x4_vertical_right_10_sse2;
347347 h->pred4x4[HOR_DOWN_PRED ] = ff_pred4x4_horizontal_down_10_sse2;
348348
349− if (chroma_format_idc == 1) {
349+ if (chroma_format_idc <= 1) {
350350 h->pred8x8[DC_PRED8x8 ] = ff_pred8x8_dc_10_sse2;
351351 h->pred8x8[TOP_DC_PRED8x8 ] = ff_pred8x8_top_dc_10_sse2;
352352 h->pred8x8[PLANE_PRED8x8 ] = ff_pred8x8_plane_10_sse2;
libavcodec/x86/h264dsp_init.c+7−7
@@ -213,7 +213,7 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth, · 212 unmodified lines
213213 {
214214 int cpu_flags = av_get_cpu_flags();
215215
216− if (chroma_format_idc == 1 && EXTERNAL_MMXEXT(cpu_flags))
216+ if (chroma_format_idc <= 1 && EXTERNAL_MMXEXT(cpu_flags))
217217 c->h264_loop_filter_strength = ff_h264_loop_filter_strength_mmxext;
218218
219219 if (bit_depth == 8) {
@@ -225,7 +225,7 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth, · 5 unmodified lines
225225
226226 c->h264_idct_add16 = ff_h264_idct_add16_8_mmx;
227227 c->h264_idct8_add4 = ff_h264_idct8_add4_8_mmx;
228− if (chroma_format_idc == 1)
228+ if (chroma_format_idc <= 1)
229229 c->h264_idct_add8 = ff_h264_idct_add8_8_mmx;
230230 c->h264_idct_add16intra = ff_h264_idct_add16intra_8_mmx;
231231 if (cpu_flags & AV_CPU_FLAG_CMOV)
@@ -236,13 +236,13 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth, · 4 unmodified lines
236236 c->h264_idct8_dc_add = ff_h264_idct8_dc_add_8_mmxext;
237237 c->h264_idct_add16 = ff_h264_idct_add16_8_mmxext;
238238 c->h264_idct8_add4 = ff_h264_idct8_add4_8_mmxext;
239− if (chroma_format_idc == 1)
239+ if (chroma_format_idc <= 1)
240240 c->h264_idct_add8 = ff_h264_idct_add8_8_mmxext;
241241 c->h264_idct_add16intra = ff_h264_idct_add16intra_8_mmxext;
242242
243243 c->h264_v_loop_filter_chroma = ff_deblock_v_chroma_8_mmxext;
244244 c->h264_v_loop_filter_chroma_intra = ff_deblock_v_chroma_intra_8_mmxext;
245− if (chroma_format_idc == 1) {
245+ if (chroma_format_idc <= 1) {
246246 c->h264_h_loop_filter_chroma = ff_deblock_h_chroma_8_mmxext;
247247 c->h264_h_loop_filter_chroma_intra = ff_deblock_h_chroma_intra_8_mmxext;
248248 }
@@ -265,7 +265,7 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth, · 16 unmodified lines
265265
266266 c->h264_idct_add16 = ff_h264_idct_add16_8_sse2;
267267 c->h264_idct8_add4 = ff_h264_idct8_add4_8_sse2;
268− if (chroma_format_idc == 1)
268+ if (chroma_format_idc <= 1)
269269 c->h264_idct_add8 = ff_h264_idct_add8_8_sse2;
270270 c->h264_idct_add16intra = ff_h264_idct_add16intra_8_sse2;
271271 c->h264_luma_dc_dequant_idct = ff_h264_luma_dc_dequant_idct_sse2;
@@ -308,7 +308,7 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth, · 36 unmodified lines
308308 c->h264_idct8_dc_add = ff_h264_idct8_dc_add_10_sse2;
309309
310310 c->h264_idct_add16 = ff_h264_idct_add16_10_sse2;
311− if (chroma_format_idc == 1)
311+ if (chroma_format_idc <= 1)
312312 c->h264_idct_add8 = ff_h264_idct_add8_10_sse2;
313313 c->h264_idct_add16intra = ff_h264_idct_add16intra_10_sse2;
314314 #if HAVE_ALIGNED_STACK
@@ -348,7 +348,7 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth, · 33 unmodified lines
348348 c->h264_idct8_dc_add = ff_h264_idct8_dc_add_10_avx;
349349
350350 c->h264_idct_add16 = ff_h264_idct_add16_10_avx;
351− if (chroma_format_idc == 1)
351+ if (chroma_format_idc <= 1)
352352 c->h264_idct_add8 = ff_h264_idct_add8_10_avx;
353353 c->h264_idct_add16intra = ff_h264_idct_add16intra_10_avx;
354354 #if HAVE_ALIGNED_STACK

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t