History

avfilter/vf_overlay: Fix double-free of AVFilterFormats on error

The query_formats function of the overlay filter tries to allocate two lists (only one in a special case) of formats which on success are attached to more permanent objects (AVFilterLinks) for storage afterwards. If attaching a list to an AVFilterLink succeeds, it is in turn owned by the AVFilterLink (or more exactly, the AVFilterLink becomes one of the common owners of the list). Yet if attaching a list to one of its links succeeds and an error happens lateron, both lists were manually freed, whic is wrong if the list is already owned by one or more links; these links' pointers to their lists will become dangling and there will be a double-free/use-after-free when these links are cleaned up automatically. This commit fixes this by removing the custom freeing code; this will temporarily add a leaking codepath (if attaching a list not already owned by a link to a link fails, the list will leak), but this will be fixed soon by making sure that an AVFilterFormats without owner will be automatically freed when attaching it to an AVFilterLink fails. Notice that at most one list leaks because a new list is only allocated after the old list has been successfully attached to a link.

Andreas Rheinhardt committed 6 years ago

Commit a86ee5f1 parent 07240c31 file · +23 −61Browse files

Showing the changes to libavfilter (1 of 1 file). Show all

Unified · Split

libavfilter/vf_overlay.c+23−61
@@ -221,90 +221,52 @@ static int query_formats(AVFilterContext *ctx) · 220 unmodified lines
221221 AV_PIX_FMT_NONE
222222 };
223223
224− AVFilterFormats *main_formats = NULL;
225− AVFilterFormats *overlay_formats = NULL;
224+ const enum AVPixelFormat *main_formats, *overlay_formats;
225+ AVFilterFormats *formats;
226226 int ret;
227227
228228 switch (s->format) {
229229 case OVERLAY_FORMAT_YUV420:
230− if (!(main_formats = ff_make_format_list(main_pix_fmts_yuv420)) ||
231− !(overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv420))) {
232− ret = AVERROR(ENOMEM);
233− goto fail;
234− }
230+ main_formats = main_pix_fmts_yuv420;
231+ overlay_formats = overlay_pix_fmts_yuv420;
235232 break;
236233 case OVERLAY_FORMAT_YUV420P10:
237− if (!(main_formats = ff_make_format_list(main_pix_fmts_yuv420p10)) ||
238− !(overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv420p10))) {
239− ret = AVERROR(ENOMEM);
240− goto fail;
241− }
234+ main_formats = main_pix_fmts_yuv420p10;
235+ overlay_formats = overlay_pix_fmts_yuv420p10;
242236 break;
243237 case OVERLAY_FORMAT_YUV422:
244− if (!(main_formats = ff_make_format_list(main_pix_fmts_yuv422)) ||
245− !(overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv422))) {
246− ret = AVERROR(ENOMEM);
247− goto fail;
248− }
238+ main_formats = main_pix_fmts_yuv422;
239+ overlay_formats = overlay_pix_fmts_yuv422;
249240 break;
250241 case OVERLAY_FORMAT_YUV422P10:
251− if (!(main_formats = ff_make_format_list(main_pix_fmts_yuv422p10)) ||
252− !(overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv422p10))) {
253− ret = AVERROR(ENOMEM);
254− goto fail;
255− }
242+ main_formats = main_pix_fmts_yuv422p10;
243+ overlay_formats = overlay_pix_fmts_yuv422p10;
256244 break;
257245 case OVERLAY_FORMAT_YUV444:
258− if (!(main_formats = ff_make_format_list(main_pix_fmts_yuv444)) ||
259− !(overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv444))) {
260− ret = AVERROR(ENOMEM);
261− goto fail;
262− }
246+ main_formats = main_pix_fmts_yuv444;
247+ overlay_formats = overlay_pix_fmts_yuv444;
263248 break;
264249 case OVERLAY_FORMAT_RGB:
265− if (!(main_formats = ff_make_format_list(main_pix_fmts_rgb)) ||
266− !(overlay_formats = ff_make_format_list(overlay_pix_fmts_rgb))) {
267− ret = AVERROR(ENOMEM);
268− goto fail;
269− }
250+ main_formats = main_pix_fmts_rgb;
251+ overlay_formats = overlay_pix_fmts_rgb;
270252 break;
271253 case OVERLAY_FORMAT_GBRP:
272− if (!(main_formats = ff_make_format_list(main_pix_fmts_gbrp)) ||
273− !(overlay_formats = ff_make_format_list(overlay_pix_fmts_gbrp))) {
274− ret = AVERROR(ENOMEM);
275− goto fail;
276− }
254+ main_formats = main_pix_fmts_gbrp;
255+ overlay_formats = overlay_pix_fmts_gbrp;
277256 break;
278257 case OVERLAY_FORMAT_AUTO:
279− if (!(main_formats = ff_make_format_list(alpha_pix_fmts))) {
280− ret = AVERROR(ENOMEM);
281− goto fail;
282− }
283− break;
258+ return ff_set_common_formats(ctx, ff_make_format_list(alpha_pix_fmts));
284259 default:
285260 av_assert0(0);
286261 }
287262
288− if (s->format == OVERLAY_FORMAT_AUTO) {
289− ret = ff_set_common_formats(ctx, main_formats);
290− if (ret < 0)
291− goto fail;
292− } else {
293− if ((ret = ff_formats_ref(main_formats , &ctx->inputs[MAIN]->out_formats )) < 0 ||
294− (ret = ff_formats_ref(overlay_formats, &ctx->inputs[OVERLAY]->out_formats)) < 0 ||
295− (ret = ff_formats_ref(main_formats , &ctx->outputs[MAIN]->in_formats )) < 0)
296− goto fail;
297− }
263+ formats = ff_make_format_list(main_formats);
264+ if ((ret = ff_formats_ref(formats, &ctx->inputs[MAIN]->out_formats)) < 0 ||
265+ (ret = ff_formats_ref(formats, &ctx->outputs[MAIN]->in_formats)) < 0)
266+ return ret;
298267
299− return 0;
300−fail:
301− if (main_formats)
302− av_freep(&main_formats->formats);
303− av_freep(&main_formats);
304− if (overlay_formats)
305− av_freep(&overlay_formats->formats);
306− av_freep(&overlay_formats);
307− return ret;
268+ return ff_formats_ref(ff_make_format_list(overlay_formats),
269+ &ctx->inputs[OVERLAY]->out_formats);
308270 }
309271
310272 static int config_input_overlay(AVFilterLink *inlink)

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t