History

avfilter/vf_overlay: Fix double-free of AVFilterFormats on error

The query_formats function of the overlay filter tries to allocate two lists (only one in a special case) of formats which on success are attached to more permanent objects (AVFilterLinks) for storage afterwards. If attaching a list to an AVFilterLink succeeds, it is in turn owned by the AVFilterLink (or more exactly, the AVFilterLink becomes one of the common owners of the list). Yet if attaching a list to one of its links succeeds and an error happens lateron, both lists were manually freed, whic is wrong if the list is already owned by one or more links; these links' pointers to their lists will become dangling and there will be a double-free/use-after-free when these links are cleaned up automatically. This commit fixes this by removing the custom freeing code; this will temporarily add a leaking codepath (if attaching a list not already owned by a link to a link fails, the list will leak), but this will be fixed soon by making sure that an AVFilterFormats without owner will be automatically freed when attaching it to an AVFilterLink fails. Notice that at most one list leaks because a new list is only allocated after the old list has been successfully attached to a link.

Andreas Rheinhardt committed 6 years ago

Commit a86ee5f1 parent 07240c31 file · +23 −61Browse files

Unified · Split

libavfilter/vf_overlay.c+23−61
@@ -221,90 +221,52 @@ static int query_formats(AVFilterContext *ctx) · 220 unmodified lines
221 AV_PIX_FMT_NONE221 AV_PIX_FMT_NONE
222 };222 };
223 223
224 AVFilterFormats *main_formats = NULL;224 const enum AVPixelFormat *main_formats, *overlay_formats;
225 AVFilterFormats *overlay_formats = NULL;225 AVFilterFormats *formats;
226 int ret;226 int ret;
227 227
228 switch (s->format) {228 switch (s->format) {
229 case OVERLAY_FORMAT_YUV420:229 case OVERLAY_FORMAT_YUV420:
230 if (!(main_formats = ff_make_format_list(main_pix_fmts_yuv420)) ||230 main_formats = main_pix_fmts_yuv420;
231 !(overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv420))) {231 overlay_formats = overlay_pix_fmts_yuv420;
232 ret = AVERROR(ENOMEM);
233 goto fail;
234 }
235 break;232 break;
236 case OVERLAY_FORMAT_YUV420P10:233 case OVERLAY_FORMAT_YUV420P10:
237 if (!(main_formats = ff_make_format_list(main_pix_fmts_yuv420p10)) ||234 main_formats = main_pix_fmts_yuv420p10;
238 !(overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv420p10))) {235 overlay_formats = overlay_pix_fmts_yuv420p10;
239 ret = AVERROR(ENOMEM);
240 goto fail;
241 }
242 break;236 break;
243 case OVERLAY_FORMAT_YUV422:237 case OVERLAY_FORMAT_YUV422:
244 if (!(main_formats = ff_make_format_list(main_pix_fmts_yuv422)) ||238 main_formats = main_pix_fmts_yuv422;
245 !(overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv422))) {239 overlay_formats = overlay_pix_fmts_yuv422;
246 ret = AVERROR(ENOMEM);
247 goto fail;
248 }
249 break;240 break;
250 case OVERLAY_FORMAT_YUV422P10:241 case OVERLAY_FORMAT_YUV422P10:
251 if (!(main_formats = ff_make_format_list(main_pix_fmts_yuv422p10)) ||242 main_formats = main_pix_fmts_yuv422p10;
252 !(overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv422p10))) {243 overlay_formats = overlay_pix_fmts_yuv422p10;
253 ret = AVERROR(ENOMEM);
254 goto fail;
255 }
256 break;244 break;
257 case OVERLAY_FORMAT_YUV444:245 case OVERLAY_FORMAT_YUV444:
258 if (!(main_formats = ff_make_format_list(main_pix_fmts_yuv444)) ||246 main_formats = main_pix_fmts_yuv444;
259 !(overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv444))) {247 overlay_formats = overlay_pix_fmts_yuv444;
260 ret = AVERROR(ENOMEM);
261 goto fail;
262 }
263 break;248 break;
264 case OVERLAY_FORMAT_RGB:249 case OVERLAY_FORMAT_RGB:
265 if (!(main_formats = ff_make_format_list(main_pix_fmts_rgb)) ||250 main_formats = main_pix_fmts_rgb;
266 !(overlay_formats = ff_make_format_list(overlay_pix_fmts_rgb))) {251 overlay_formats = overlay_pix_fmts_rgb;
267 ret = AVERROR(ENOMEM);
268 goto fail;
269 }
270 break;252 break;
271 case OVERLAY_FORMAT_GBRP:253 case OVERLAY_FORMAT_GBRP:
272 if (!(main_formats = ff_make_format_list(main_pix_fmts_gbrp)) ||254 main_formats = main_pix_fmts_gbrp;
273 !(overlay_formats = ff_make_format_list(overlay_pix_fmts_gbrp))) {255 overlay_formats = overlay_pix_fmts_gbrp;
274 ret = AVERROR(ENOMEM);
275 goto fail;
276 }
277 break;256 break;
278 case OVERLAY_FORMAT_AUTO:257 case OVERLAY_FORMAT_AUTO:
279 if (!(main_formats = ff_make_format_list(alpha_pix_fmts))) {258 return ff_set_common_formats(ctx, ff_make_format_list(alpha_pix_fmts));
280 ret = AVERROR(ENOMEM);
281 goto fail;
282 }
283 break;
284 default:259 default:
285 av_assert0(0);260 av_assert0(0);
286 }261 }
287 262
288 if (s->format == OVERLAY_FORMAT_AUTO) {263 formats = ff_make_format_list(main_formats);
289 ret = ff_set_common_formats(ctx, main_formats);264 if ((ret = ff_formats_ref(formats, &ctx->inputs[MAIN]->out_formats)) < 0 ||
290 if (ret < 0)265 (ret = ff_formats_ref(formats, &ctx->outputs[MAIN]->in_formats)) < 0)
291 goto fail;266 return ret;
292 } else {
293 if ((ret = ff_formats_ref(main_formats , &ctx->inputs[MAIN]->out_formats )) < 0 ||
294 (ret = ff_formats_ref(overlay_formats, &ctx->inputs[OVERLAY]->out_formats)) < 0 ||
295 (ret = ff_formats_ref(main_formats , &ctx->outputs[MAIN]->in_formats )) < 0)
296 goto fail;
297 }
298 267
299 return 0;268 return ff_formats_ref(ff_make_format_list(overlay_formats),
300fail:269 &ctx->inputs[OVERLAY]->out_formats);
301 if (main_formats)
302 av_freep(&main_formats->formats);
303 av_freep(&main_formats);
304 if (overlay_formats)
305 av_freep(&overlay_formats->formats);
306 av_freep(&overlay_formats);
307 return ret;
308}270}
309 271
310static int config_input_overlay(AVFilterLink *inlink)272static int config_input_overlay(AVFilterLink *inlink)

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t