This filter allows crop detection even if the video is embedded in non-black areas.
Showing the changes to libavfilter (2 of 6 files). Show all
Unified · Split
| @@ -32,7 +32,7 @@ · 31 unmodified lines | ||
| 32 | 32 | #include "version_major.h" |
| 33 | 33 | |
| 34 | 34 | #define LIBAVFILTER_VERSION_MINOR 46 |
| 35 | −#define LIBAVFILTER_VERSION_MICRO 100 | |
| 35 | +#define LIBAVFILTER_VERSION_MICRO 101 | |
| 36 | 36 | |
| 37 | 37 | |
| 38 | 38 | #define LIBAVFILTER_VERSION_INT AV_VERSION_INT(LIBAVFILTER_VERSION_MAJOR, \ |
| @@ -26,11 +26,14 @@ · 25 unmodified lines | ||
| 26 | 26 | #include "libavutil/imgutils.h" |
| 27 | 27 | #include "libavutil/internal.h" |
| 28 | 28 | #include "libavutil/opt.h" |
| 29 | +#include "libavutil/motion_vector.h" | |
| 30 | +#include "libavutil/qsort.h" | |
| 29 | 31 | |
| 30 | 32 | #include "avfilter.h" |
| 31 | 33 | #include "formats.h" |
| 32 | 34 | #include "internal.h" |
| 33 | 35 | #include "video.h" |
| 36 | +#include "edge_common.h" | |
| 34 | 37 | |
| 35 | 38 | typedef struct CropDetectContext { |
| 36 | 39 | const AVClass *class; |
| @@ -42,6 +45,16 @@ typedef struct CropDetectContext { · 5 unmodified lines | ||
| 42 | 45 | int frame_nb; |
| 43 | 46 | int max_pixsteps[4]; |
| 44 | 47 | int max_outliers; |
| 48 | + int mode; | |
| 49 | + int window_size; | |
| 50 | + int mv_threshold; | |
| 51 | + float low, high; | |
| 52 | + uint8_t low_u8, high_u8; | |
| 53 | + uint8_t *filterbuf; | |
| 54 | + uint8_t *tmpbuf; | |
| 55 | + uint16_t *gradients; | |
| 56 | + char *directions; | |
| 57 | + int *bboxes[4]; | |
| 45 | 58 | } CropDetectContext; |
| 46 | 59 | |
| 47 | 60 | static const enum AVPixelFormat pix_fmts[] = { |
| @@ -61,6 +74,17 @@ static const enum AVPixelFormat pix_fmts[] = { · 13 unmodified lines | ||
| 61 | 74 | AV_PIX_FMT_NONE |
| 62 | 75 | }; |
| 63 | 76 | |
| 77 | +enum CropMode { | |
| 78 | + MODE_BLACK, | |
| 79 | + MODE_MV_EDGES, | |
| 80 | + MODE_NB | |
| 81 | +}; | |
| 82 | + | |
| 83 | +static int comp(const int *a,const int *b) | |
| 84 | +{ | |
| 85 | + return FFDIFFSIGN(*a, *b); | |
| 86 | +} | |
| 87 | + | |
| 64 | 88 | static int checkline(void *ctx, const unsigned char *src, int stride, int len, int bpp) |
| 65 | 89 | { |
| 66 | 90 | int total = 0; |
| @@ -116,11 +140,43 @@ static int checkline(void *ctx, const unsigned char *src, int stride, int len, i · 49 unmodified lines | ||
| 116 | 140 | return total; |
| 117 | 141 | } |
| 118 | 142 | |
| 143 | +static int checkline_edge(void *ctx, const unsigned char *src, int stride, int len, int bpp) | |
| 144 | +{ | |
| 145 | + const uint16_t *src16 = (const uint16_t *)src; | |
| 146 | + | |
| 147 | + switch (bpp) { | |
| 148 | + case 1: | |
| 149 | + while (--len >= 0) { | |
| 150 | + if (src[0]) return 0; | |
| 151 | + src += stride; | |
| 152 | + } | |
| 153 | + break; | |
| 154 | + case 2: | |
| 155 | + stride >>= 1; | |
| 156 | + while (--len >= 0) { | |
| 157 | + if (src16[0]) return 0; | |
| 158 | + src16 += stride; | |
| 159 | + } | |
| 160 | + break; | |
| 161 | + case 3: | |
| 162 | + case 4: | |
| 163 | + while (--len >= 0) { | |
| 164 | + if (src[0] || src[1] || src[2]) return 0; | |
| 165 | + src += stride; | |
| 166 | + } | |
| 167 | + break; | |
| 168 | + } | |
| 169 | + | |
| 170 | + return 1; | |
| 171 | +} | |
| 172 | + | |
| 119 | 173 | static av_cold int init(AVFilterContext *ctx) |
| 120 | 174 | { |
| 121 | 175 | CropDetectContext *s = ctx->priv; |
| 122 | 176 | |
| 123 | 177 | s->frame_nb = -1 * s->skip; |
| 178 | + s->low_u8 = s->low * 255. + .5; | |
| 179 | + s->high_u8 = s->high * 255. + .5; | |
| 124 | 180 | |
| 125 | 181 | av_log(ctx, AV_LOG_VERBOSE, "limit:%f round:%d skip:%d reset_count:%d\n", |
| 126 | 182 | s->limit, s->round, s->skip, s->reset_count); |
| @@ -128,11 +184,27 @@ static av_cold int init(AVFilterContext *ctx) · 1 unmodified line | ||
| 128 | 184 | return 0; |
| 129 | 185 | } |
| 130 | 186 | |
| 187 | +static av_cold void uninit(AVFilterContext *ctx) | |
| 188 | +{ | |
| 189 | + CropDetectContext *s = ctx->priv; | |
| 190 | + | |
| 191 | + av_freep(&s->tmpbuf); | |
| 192 | + av_freep(&s->filterbuf); | |
| 193 | + av_freep(&s->gradients); | |
| 194 | + av_freep(&s->directions); | |
| 195 | + av_freep(&s->bboxes[0]); | |
| 196 | + av_freep(&s->bboxes[1]); | |
| 197 | + av_freep(&s->bboxes[2]); | |
| 198 | + av_freep(&s->bboxes[3]); | |
| 199 | +} | |
| 200 | + | |
| 131 | 201 | static int config_input(AVFilterLink *inlink) |
| 132 | 202 | { |
| 133 | 203 | AVFilterContext *ctx = inlink->dst; |
| 134 | 204 | CropDetectContext *s = ctx->priv; |
| 135 | 205 | const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format); |
| 206 | + const int bufsize = inlink->w * inlink->h; | |
| 207 | + int bpp; | |
| 136 | 208 | |
| 137 | 209 | av_image_fill_max_pixsteps(s->max_pixsteps, NULL, desc); |
| 138 | 210 | |
| @@ -144,6 +216,21 @@ static int config_input(AVFilterLink *inlink) · 5 unmodified lines | ||
| 144 | 216 | s->x2 = 0; |
| 145 | 217 | s->y2 = 0; |
| 146 | 218 | |
| 219 | + bpp = s->max_pixsteps[0]; | |
| 220 | + s->window_size = FFMAX(s->reset_count, 15); | |
| 221 | + s->tmpbuf = av_malloc(bufsize); | |
| 222 | + s->filterbuf = av_malloc(bufsize * s->max_pixsteps[0]); | |
| 223 | + s->gradients = av_calloc(bufsize, sizeof(*s->gradients)); | |
| 224 | + s->directions = av_malloc(bufsize); | |
| 225 | + s->bboxes[0] = av_malloc(s->window_size * sizeof(*s->bboxes[0])); | |
| 226 | + s->bboxes[1] = av_malloc(s->window_size * sizeof(*s->bboxes[1])); | |
| 227 | + s->bboxes[2] = av_malloc(s->window_size * sizeof(*s->bboxes[2])); | |
| 228 | + s->bboxes[3] = av_malloc(s->window_size * sizeof(*s->bboxes[3])); | |
| 229 | + | |
| 230 | + if (!s->tmpbuf || !s->filterbuf || !s->gradients || !s->directions || | |
| 231 | + !s->bboxes[0] || !s->bboxes[1] || !s->bboxes[2] || !s->bboxes[3]) | |
| 232 | + return AVERROR(ENOMEM); | |
| 233 | + | |
| 147 | 234 | return 0; |
| 148 | 235 | } |
| 149 | 236 | |
| @@ -155,11 +242,28 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *frame) · 5 unmodified lines | ||
| 155 | 242 | AVFilterContext *ctx = inlink->dst; |
| 156 | 243 | CropDetectContext *s = ctx->priv; |
| 157 | 244 | int bpp = s->max_pixsteps[0]; |
| 158 | − int w, h, x, y, shrink_by; | |
| 245 | + int w, h, x, y, shrink_by, i; | |
| 159 | 246 | AVDictionary **metadata; |
| 160 | 247 | int outliers, last_y; |
| 161 | 248 | int limit = lrint(s->limit); |
| 162 | 249 | |
| 250 | + const int inw = inlink->w; | |
| 251 | + const int inh = inlink->h; | |
| 252 | + uint8_t *tmpbuf = s->tmpbuf; | |
| 253 | + uint8_t *filterbuf = s->filterbuf; | |
| 254 | + uint16_t *gradients = s->gradients; | |
| 255 | + int8_t *directions = s->directions; | |
| 256 | + const AVFrameSideData *sd = NULL; | |
| 257 | + int scan_w, scan_h, bboff; | |
| 258 | + | |
| 259 | + void (*sobel)(int w, int h, uint16_t *dst, int dst_linesize, | |
| 260 | + int8_t *dir, int dir_linesize, | |
| 261 | + const uint8_t *src, int src_linesize, int src_stride) = (bpp == 2) ? &ff_sobel_16 : &ff_sobel_8; | |
| 262 | + void (*gaussian_blur)(int w, int h, | |
| 263 | + uint8_t *dst, int dst_linesize, | |
| 264 | + const uint8_t *src, int src_linesize, int src_stride) = (bpp == 2) ? &ff_gaussian_blur_16 : &ff_gaussian_blur_8; | |
| 265 | + | |
| 266 | + | |
| 163 | 267 | // ignore first s->skip frames |
| 164 | 268 | if (++s->frame_nb > 0) { |
| 165 | 269 | metadata = &frame->metadata; |
| @@ -185,11 +289,109 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *frame) · 19 unmodified lines | ||
| 185 | 289 | last_y = y INC;\ |
| 186 | 290 | } |
| 187 | 291 | |
| 292 | + if (s->mode == MODE_BLACK) { | |
| 188 | 293 | FIND(s->y1, 0, y < s->y1, +1, frame->linesize[0], bpp, frame->width); |
| 189 | 294 | FIND(s->y2, frame->height - 1, y > FFMAX(s->y2, s->y1), -1, frame->linesize[0], bpp, frame->width); |
| 190 | 295 | FIND(s->x1, 0, y < s->x1, +1, bpp, frame->linesize[0], frame->height); |
| 191 | 296 | FIND(s->x2, frame->width - 1, y > FFMAX(s->x2, s->x1), -1, bpp, frame->linesize[0], frame->height); |
| 297 | + } else { // MODE_MV_EDGES | |
| 298 | + sd = av_frame_get_side_data(frame, AV_FRAME_DATA_MOTION_VECTORS); | |
| 299 | + s->x1 = 0; | |
| 300 | + s->y1 = 0; | |
| 301 | + s->x2 = inw - 1; | |
| 302 | + s->y2 = inh - 1; | |
| 303 | + | |
| 304 | + if (!sd) { | |
| 305 | + av_log(ctx, AV_LOG_WARNING, "Cannot detect: no motion vectors available"); | |
| 306 | + } else { | |
| 307 | + // gaussian filter to reduce noise | |
| 308 | + gaussian_blur(inw, inh, | |
| 309 | + filterbuf, inw*bpp, | |
| 310 | + frame->data[0], frame->linesize[0], bpp); | |
| 311 | + | |
| 312 | + // compute the 16-bits gradients and directions for the next step | |
| 313 | + sobel(inw, inh, gradients, inw, directions, inw, filterbuf, inw*bpp, bpp); | |
| 314 | + | |
| 315 | + // non_maximum_suppression() will actually keep & clip what's necessary and | |
| 316 | + // ignore the rest, so we need a clean output buffer | |
| 317 | + memset(tmpbuf, 0, inw * inh); | |
| 318 | + ff_non_maximum_suppression(inw, inh, tmpbuf, inw, directions, inw, gradients, inw); | |
| 319 | + | |
| 320 | + | |
| 321 | + // keep high values, or low values surrounded by high values | |
| 322 | + ff_double_threshold(s->low_u8, s->high_u8, inw, inh, | |
| 323 | + tmpbuf, inw, tmpbuf, inw); | |
| 324 | + | |
| 325 | + // scan all MVs and store bounding box | |
| 326 | + s->x1 = inw - 1; | |
| 327 | + s->y1 = inh - 1; | |
| 328 | + s->x2 = 0; | |
| 329 | + s->y2 = 0; | |
| 330 | + for (i = 0; i < sd->size / sizeof(AVMotionVector); i++) { | |
| 331 | + const AVMotionVector *mv = (const AVMotionVector*)sd->data + i; | |
| 332 | + const int mx = mv->dst_x - mv->src_x; | |
| 333 | + const int my = mv->dst_y - mv->src_y; | |
| 334 | + | |
| 335 | + if (mv->dst_x >= 0 && mv->dst_x < inw && | |
| 336 | + mv->dst_y >= 0 && mv->dst_y < inh && | |
| 337 | + mv->src_x >= 0 && mv->src_x < inw && | |
| 338 | + mv->src_y >= 0 && mv->src_y < inh && | |
| 339 | + mx * mx + my * my >= s->mv_threshold * s->mv_threshold) { | |
| 340 | + s->x1 = mv->dst_x < s->x1 ? mv->dst_x : s->x1; | |
| 341 | + s->y1 = mv->dst_y < s->y1 ? mv->dst_y : s->y1; | |
| 342 | + s->x2 = mv->dst_x > s->x2 ? mv->dst_x : s->x2; | |
| 343 | + s->y2 = mv->dst_y > s->y2 ? mv->dst_y : s->y2; | |
| 344 | + } | |
| 345 | + } | |
| 346 | + | |
| 347 | + // assert x1<x2, y1<y2 | |
| 348 | + if (s->x1 > s->x2) FFSWAP(int, s->x1, s->x2); | |
| 349 | + if (s->y1 > s->y2) FFSWAP(int, s->y1, s->y2); | |
| 350 | + | |
| 351 | + // scan outward looking for 0-edge-lines in edge image | |
| 352 | + scan_w = s->x2 - s->x1; | |
| 353 | + scan_h = s->y2 - s->y1; | |
| 354 | + | |
| 355 | +#define FIND_EDGE(DST, FROM, NOEND, INC, STEP0, STEP1, LEN) \ | |
| 356 | + for (last_y = y = FROM; NOEND; y = y INC) { \ | |
| 357 | + if (checkline_edge(ctx, tmpbuf + STEP0 * y, STEP1, LEN, bpp)) { \ | |
| 358 | + if (last_y INC == y) { \ | |
| 359 | + DST = y; \ | |
| 360 | + break; \ | |
| 361 | + } else \ | |
| 362 | + last_y = y; \ | |
| 363 | + } \ | |
| 364 | + } \ | |
| 365 | + if (!(NOEND)) { \ | |
| 366 | + DST = y -(INC); \ | |
| 367 | + } | |
| 192 | 368 | |
| 369 | + FIND_EDGE(s->y1, s->y1, y >= 0, -1, inw, bpp, scan_w); | |
| 370 | + FIND_EDGE(s->y2, s->y2, y < inh, +1, inw, bpp, scan_w); | |
| 371 | + FIND_EDGE(s->x1, s->x1, y >= 0, -1, bpp, inw, scan_h); | |
| 372 | + FIND_EDGE(s->x2, s->x2, y < inw, +1, bpp, inw, scan_h); | |
| 373 | + | |
| 374 | + // queue bboxes | |
| 375 | + bboff = (s->frame_nb - 1) % s->window_size; | |
| 376 | + s->bboxes[0][bboff] = s->x1; | |
| 377 | + s->bboxes[1][bboff] = s->x2; | |
| 378 | + s->bboxes[2][bboff] = s->y1; | |
| 379 | + s->bboxes[3][bboff] = s->y2; | |
| 380 | + | |
| 381 | + // sort queue | |
| 382 | + bboff = FFMIN(s->frame_nb, s->window_size); | |
| 383 | + AV_QSORT(s->bboxes[0], bboff, int, comp); | |
| 384 | + AV_QSORT(s->bboxes[1], bboff, int, comp); | |
| 385 | + AV_QSORT(s->bboxes[2], bboff, int, comp); | |
| 386 | + AV_QSORT(s->bboxes[3], bboff, int, comp); | |
| 387 | + | |
| 388 | + // return median of window_size elems | |
| 389 | + s->x1 = s->bboxes[0][bboff/2]; | |
| 390 | + s->x2 = s->bboxes[1][bboff/2]; | |
| 391 | + s->y1 = s->bboxes[2][bboff/2]; | |
| 392 | + s->y2 = s->bboxes[3][bboff/2]; | |
| 393 | + } | |
| 394 | + } | |
| 193 | 395 | |
| 194 | 396 | // round x and y (up), important for yuv colorspaces |
| 195 | 397 | // make sure they stay rounded! |
| @@ -243,6 +445,12 @@ static const AVOption cropdetect_options[] = { · 47 unmodified lines | ||
| 243 | 445 | { "skip", "Number of initial frames to skip", OFFSET(skip), AV_OPT_TYPE_INT, { .i64 = 2 }, 0, INT_MAX, FLAGS }, |
| 244 | 446 | { "reset_count", "Recalculate the crop area after this many frames",OFFSET(reset_count),AV_OPT_TYPE_INT,{ .i64 = 0 }, 0, INT_MAX, FLAGS }, |
| 245 | 447 | { "max_outliers", "Threshold count of outliers", OFFSET(max_outliers),AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, FLAGS }, |
| 448 | + { "mode", "set mode", OFFSET(mode), AV_OPT_TYPE_INT, {.i64=MODE_BLACK}, 0, MODE_NB-1, FLAGS, "mode" }, | |
| 449 | + { "black", "detect black pixels surrounding the video", 0, AV_OPT_TYPE_CONST, {.i64=MODE_BLACK}, INT_MIN, INT_MAX, FLAGS, "mode" }, | |
| 450 | + { "mvedges", "detect motion and edged surrounding the video", 0, AV_OPT_TYPE_CONST, {.i64=MODE_MV_EDGES}, INT_MIN, INT_MAX, FLAGS, "mode" }, | |
| 451 | + { "high", "Set high threshold for edge detection", OFFSET(high), AV_OPT_TYPE_FLOAT, {.dbl=25/255.}, 0, 1, FLAGS }, | |
| 452 | + { "low", "Set low threshold for edge detection", OFFSET(low), AV_OPT_TYPE_FLOAT, {.dbl=15/255.}, 0, 1, FLAGS }, | |
| 453 | + { "mv_threshold", "motion vector threshold when estimating video window size", OFFSET(mv_threshold), AV_OPT_TYPE_INT, {.i64=8}, 0, 100, FLAGS}, | |
| 246 | 454 | { NULL } |
| 247 | 455 | }; |
| 248 | 456 | |
| @@ -270,6 +478,7 @@ const AVFilter ff_vf_cropdetect = { · 21 unmodified lines | ||
| 270 | 478 | .priv_size = sizeof(CropDetectContext), |
| 271 | 479 | .priv_class = &cropdetect_class, |
| 272 | 480 | .init = init, |
| 481 | + .uninit = uninit, | |
| 273 | 482 | FILTER_INPUTS(avfilter_vf_cropdetect_inputs), |
| 274 | 483 | FILTER_OUTPUTS(avfilter_vf_cropdetect_outputs), |
| 275 | 484 | FILTER_PIXFMTS_ARRAY(pix_fmts), |