| @@ -26,11 +26,14 @@ · 25 unmodified lines |
| 26 | #include "libavutil/imgutils.h" | 26 | #include "libavutil/imgutils.h" |
| 27 | #include "libavutil/internal.h" | 27 | #include "libavutil/internal.h" |
| 28 | #include "libavutil/opt.h" | 28 | #include "libavutil/opt.h" |
| | 29 | #include "libavutil/motion_vector.h" |
| | 30 | #include "libavutil/qsort.h" |
| 29 | | 31 | |
| 30 | #include "avfilter.h" | 32 | #include "avfilter.h" |
| 31 | #include "formats.h" | 33 | #include "formats.h" |
| 32 | #include "internal.h" | 34 | #include "internal.h" |
| 33 | #include "video.h" | 35 | #include "video.h" |
| | 36 | #include "edge_common.h" |
| 34 | | 37 | |
| 35 | typedef struct CropDetectContext { | 38 | typedef struct CropDetectContext { |
| 36 | const AVClass *class; | 39 | const AVClass *class; |
| @@ -42,6 +45,16 @@ typedef struct CropDetectContext { · 5 unmodified lines |
| 42 | int frame_nb; | 45 | int frame_nb; |
| 43 | int max_pixsteps[4]; | 46 | int max_pixsteps[4]; |
| 44 | int max_outliers; | 47 | int max_outliers; |
| | 48 | int mode; |
| | 49 | int window_size; |
| | 50 | int mv_threshold; |
| | 51 | float low, high; |
| | 52 | uint8_t low_u8, high_u8; |
| | 53 | uint8_t *filterbuf; |
| | 54 | uint8_t *tmpbuf; |
| | 55 | uint16_t *gradients; |
| | 56 | char *directions; |
| | 57 | int *bboxes[4]; |
| 45 | } CropDetectContext; | 58 | } CropDetectContext; |
| 46 | | 59 | |
| 47 | static const enum AVPixelFormat pix_fmts[] = { | 60 | static const enum AVPixelFormat pix_fmts[] = { |
| @@ -61,6 +74,17 @@ static const enum AVPixelFormat pix_fmts[] = { · 13 unmodified lines |
| 61 | AV_PIX_FMT_NONE | 74 | AV_PIX_FMT_NONE |
| 62 | }; | 75 | }; |
| 63 | | 76 | |
| | 77 | enum CropMode { |
| | 78 | MODE_BLACK, |
| | 79 | MODE_MV_EDGES, |
| | 80 | MODE_NB |
| | 81 | }; |
| | 82 | |
| | 83 | static int comp(const int *a,const int *b) |
| | 84 | { |
| | 85 | return FFDIFFSIGN(*a, *b); |
| | 86 | } |
| | 87 | |
| 64 | static int checkline(void *ctx, const unsigned char *src, int stride, int len, int bpp) | 88 | static int checkline(void *ctx, const unsigned char *src, int stride, int len, int bpp) |
| 65 | { | 89 | { |
| 66 | int total = 0; | 90 | int total = 0; |
| @@ -116,11 +140,43 @@ static int checkline(void *ctx, const unsigned char *src, int stride, int len, i · 49 unmodified lines |
| 116 | return total; | 140 | return total; |
| 117 | } | 141 | } |
| 118 | | 142 | |
| | 143 | static int checkline_edge(void *ctx, const unsigned char *src, int stride, int len, int bpp) |
| | 144 | { |
| | 145 | const uint16_t *src16 = (const uint16_t *)src; |
| | 146 | |
| | 147 | switch (bpp) { |
| | 148 | case 1: |
| | 149 | while (--len >= 0) { |
| | 150 | if (src[0]) return 0; |
| | 151 | src += stride; |
| | 152 | } |
| | 153 | break; |
| | 154 | case 2: |
| | 155 | stride >>= 1; |
| | 156 | while (--len >= 0) { |
| | 157 | if (src16[0]) return 0; |
| | 158 | src16 += stride; |
| | 159 | } |
| | 160 | break; |
| | 161 | case 3: |
| | 162 | case 4: |
| | 163 | while (--len >= 0) { |
| | 164 | if (src[0] || src[1] || src[2]) return 0; |
| | 165 | src += stride; |
| | 166 | } |
| | 167 | break; |
| | 168 | } |
| | 169 | |
| | 170 | return 1; |
| | 171 | } |
| | 172 | |
| 119 | static av_cold int init(AVFilterContext *ctx) | 173 | static av_cold int init(AVFilterContext *ctx) |
| 120 | { | 174 | { |
| 121 | CropDetectContext *s = ctx->priv; | 175 | CropDetectContext *s = ctx->priv; |
| 122 | | 176 | |
| 123 | s->frame_nb = -1 * s->skip; | 177 | s->frame_nb = -1 * s->skip; |
| | 178 | s->low_u8 = s->low * 255. + .5; |
| | 179 | s->high_u8 = s->high * 255. + .5; |
| 124 | | 180 | |
| 125 | av_log(ctx, AV_LOG_VERBOSE, "limit:%f round:%d skip:%d reset_count:%d\n", | 181 | av_log(ctx, AV_LOG_VERBOSE, "limit:%f round:%d skip:%d reset_count:%d\n", |
| 126 | s->limit, s->round, s->skip, s->reset_count); | 182 | s->limit, s->round, s->skip, s->reset_count); |
| @@ -128,11 +184,27 @@ static av_cold int init(AVFilterContext *ctx) · 1 unmodified line |
| 128 | return 0; | 184 | return 0; |
| 129 | } | 185 | } |
| 130 | | 186 | |
| | 187 | static av_cold void uninit(AVFilterContext *ctx) |
| | 188 | { |
| | 189 | CropDetectContext *s = ctx->priv; |
| | 190 | |
| | 191 | av_freep(&s->tmpbuf); |
| | 192 | av_freep(&s->filterbuf); |
| | 193 | av_freep(&s->gradients); |
| | 194 | av_freep(&s->directions); |
| | 195 | av_freep(&s->bboxes[0]); |
| | 196 | av_freep(&s->bboxes[1]); |
| | 197 | av_freep(&s->bboxes[2]); |
| | 198 | av_freep(&s->bboxes[3]); |
| | 199 | } |
| | 200 | |
| 131 | static int config_input(AVFilterLink *inlink) | 201 | static int config_input(AVFilterLink *inlink) |
| 132 | { | 202 | { |
| 133 | AVFilterContext *ctx = inlink->dst; | 203 | AVFilterContext *ctx = inlink->dst; |
| 134 | CropDetectContext *s = ctx->priv; | 204 | CropDetectContext *s = ctx->priv; |
| 135 | const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format); | 205 | const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format); |
| | 206 | const int bufsize = inlink->w * inlink->h; |
| | 207 | int bpp; |
| 136 | | 208 | |
| 137 | av_image_fill_max_pixsteps(s->max_pixsteps, NULL, desc); | 209 | av_image_fill_max_pixsteps(s->max_pixsteps, NULL, desc); |
| 138 | | 210 | |
| @@ -144,6 +216,21 @@ static int config_input(AVFilterLink *inlink) · 5 unmodified lines |
| 144 | s->x2 = 0; | 216 | s->x2 = 0; |
| 145 | s->y2 = 0; | 217 | s->y2 = 0; |
| 146 | | 218 | |
| | 219 | bpp = s->max_pixsteps[0]; |
| | 220 | s->window_size = FFMAX(s->reset_count, 15); |
| | 221 | s->tmpbuf = av_malloc(bufsize); |
| | 222 | s->filterbuf = av_malloc(bufsize * s->max_pixsteps[0]); |
| | 223 | s->gradients = av_calloc(bufsize, sizeof(*s->gradients)); |
| | 224 | s->directions = av_malloc(bufsize); |
| | 225 | s->bboxes[0] = av_malloc(s->window_size * sizeof(*s->bboxes[0])); |
| | 226 | s->bboxes[1] = av_malloc(s->window_size * sizeof(*s->bboxes[1])); |
| | 227 | s->bboxes[2] = av_malloc(s->window_size * sizeof(*s->bboxes[2])); |
| | 228 | s->bboxes[3] = av_malloc(s->window_size * sizeof(*s->bboxes[3])); |
| | 229 | |
| | 230 | if (!s->tmpbuf || !s->filterbuf || !s->gradients || !s->directions || |
| | 231 | !s->bboxes[0] || !s->bboxes[1] || !s->bboxes[2] || !s->bboxes[3]) |
| | 232 | return AVERROR(ENOMEM); |
| | 233 | |
| 147 | return 0; | 234 | return 0; |
| 148 | } | 235 | } |
| 149 | | 236 | |
| @@ -155,11 +242,28 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *frame) · 5 unmodified lines |
| 155 | AVFilterContext *ctx = inlink->dst; | 242 | AVFilterContext *ctx = inlink->dst; |
| 156 | CropDetectContext *s = ctx->priv; | 243 | CropDetectContext *s = ctx->priv; |
| 157 | int bpp = s->max_pixsteps[0]; | 244 | int bpp = s->max_pixsteps[0]; |
| 158 | int w, h, x, y, shrink_by; | 245 | int w, h, x, y, shrink_by, i; |
| 159 | AVDictionary **metadata; | 246 | AVDictionary **metadata; |
| 160 | int outliers, last_y; | 247 | int outliers, last_y; |
| 161 | int limit = lrint(s->limit); | 248 | int limit = lrint(s->limit); |
| 162 | | 249 | |
| | 250 | const int inw = inlink->w; |
| | 251 | const int inh = inlink->h; |
| | 252 | uint8_t *tmpbuf = s->tmpbuf; |
| | 253 | uint8_t *filterbuf = s->filterbuf; |
| | 254 | uint16_t *gradients = s->gradients; |
| | 255 | int8_t *directions = s->directions; |
| | 256 | const AVFrameSideData *sd = NULL; |
| | 257 | int scan_w, scan_h, bboff; |
| | 258 | |
| | 259 | void (*sobel)(int w, int h, uint16_t *dst, int dst_linesize, |
| | 260 | int8_t *dir, int dir_linesize, |
| | 261 | const uint8_t *src, int src_linesize, int src_stride) = (bpp == 2) ? &ff_sobel_16 : &ff_sobel_8; |
| | 262 | void (*gaussian_blur)(int w, int h, |
| | 263 | uint8_t *dst, int dst_linesize, |
| | 264 | const uint8_t *src, int src_linesize, int src_stride) = (bpp == 2) ? &ff_gaussian_blur_16 : &ff_gaussian_blur_8; |
| | 265 | |
| | 266 | |
| 163 | // ignore first s->skip frames | 267 | // ignore first s->skip frames |
| 164 | if (++s->frame_nb > 0) { | 268 | if (++s->frame_nb > 0) { |
| 165 | metadata = &frame->metadata; | 269 | metadata = &frame->metadata; |
| @@ -185,11 +289,109 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *frame) · 19 unmodified lines |
| 185 | last_y = y INC;\ | 289 | last_y = y INC;\ |
| 186 | } | 290 | } |
| 187 | | 291 | |
| | 292 | if (s->mode == MODE_BLACK) { |
| 188 | FIND(s->y1, 0, y < s->y1, +1, frame->linesize[0], bpp, frame->width); | 293 | FIND(s->y1, 0, y < s->y1, +1, frame->linesize[0], bpp, frame->width); |
| 189 | FIND(s->y2, frame->height - 1, y > FFMAX(s->y2, s->y1), -1, frame->linesize[0], bpp, frame->width); | 294 | FIND(s->y2, frame->height - 1, y > FFMAX(s->y2, s->y1), -1, frame->linesize[0], bpp, frame->width); |
| 190 | FIND(s->x1, 0, y < s->x1, +1, bpp, frame->linesize[0], frame->height); | 295 | FIND(s->x1, 0, y < s->x1, +1, bpp, frame->linesize[0], frame->height); |
| 191 | FIND(s->x2, frame->width - 1, y > FFMAX(s->x2, s->x1), -1, bpp, frame->linesize[0], frame->height); | 296 | FIND(s->x2, frame->width - 1, y > FFMAX(s->x2, s->x1), -1, bpp, frame->linesize[0], frame->height); |
| | 297 | } else { // MODE_MV_EDGES |
| | 298 | sd = av_frame_get_side_data(frame, AV_FRAME_DATA_MOTION_VECTORS); |
| | 299 | s->x1 = 0; |
| | 300 | s->y1 = 0; |
| | 301 | s->x2 = inw - 1; |
| | 302 | s->y2 = inh - 1; |
| | 303 | |
| | 304 | if (!sd) { |
| | 305 | av_log(ctx, AV_LOG_WARNING, "Cannot detect: no motion vectors available"); |
| | 306 | } else { |
| | 307 | // gaussian filter to reduce noise |
| | 308 | gaussian_blur(inw, inh, |
| | 309 | filterbuf, inw*bpp, |
| | 310 | frame->data[0], frame->linesize[0], bpp); |
| | 311 | |
| | 312 | // compute the 16-bits gradients and directions for the next step |
| | 313 | sobel(inw, inh, gradients, inw, directions, inw, filterbuf, inw*bpp, bpp); |
| | 314 | |
| | 315 | // non_maximum_suppression() will actually keep & clip what's necessary and |
| | 316 | // ignore the rest, so we need a clean output buffer |
| | 317 | memset(tmpbuf, 0, inw * inh); |
| | 318 | ff_non_maximum_suppression(inw, inh, tmpbuf, inw, directions, inw, gradients, inw); |
| | 319 | |
| | 320 | |
| | 321 | // keep high values, or low values surrounded by high values |
| | 322 | ff_double_threshold(s->low_u8, s->high_u8, inw, inh, |
| | 323 | tmpbuf, inw, tmpbuf, inw); |
| | 324 | |
| | 325 | // scan all MVs and store bounding box |
| | 326 | s->x1 = inw - 1; |
| | 327 | s->y1 = inh - 1; |
| | 328 | s->x2 = 0; |
| | 329 | s->y2 = 0; |
| | 330 | for (i = 0; i < sd->size / sizeof(AVMotionVector); i++) { |
| | 331 | const AVMotionVector *mv = (const AVMotionVector*)sd->data + i; |
| | 332 | const int mx = mv->dst_x - mv->src_x; |
| | 333 | const int my = mv->dst_y - mv->src_y; |
| | 334 | |
| | 335 | if (mv->dst_x >= 0 && mv->dst_x < inw && |
| | 336 | mv->dst_y >= 0 && mv->dst_y < inh && |
| | 337 | mv->src_x >= 0 && mv->src_x < inw && |
| | 338 | mv->src_y >= 0 && mv->src_y < inh && |
| | 339 | mx * mx + my * my >= s->mv_threshold * s->mv_threshold) { |
| | 340 | s->x1 = mv->dst_x < s->x1 ? mv->dst_x : s->x1; |
| | 341 | s->y1 = mv->dst_y < s->y1 ? mv->dst_y : s->y1; |
| | 342 | s->x2 = mv->dst_x > s->x2 ? mv->dst_x : s->x2; |
| | 343 | s->y2 = mv->dst_y > s->y2 ? mv->dst_y : s->y2; |
| | 344 | } |
| | 345 | } |
| | 346 | |
| | 347 | // assert x1<x2, y1<y2 |
| | 348 | if (s->x1 > s->x2) FFSWAP(int, s->x1, s->x2); |
| | 349 | if (s->y1 > s->y2) FFSWAP(int, s->y1, s->y2); |
| | 350 | |
| | 351 | // scan outward looking for 0-edge-lines in edge image |
| | 352 | scan_w = s->x2 - s->x1; |
| | 353 | scan_h = s->y2 - s->y1; |
| | 354 | |
| | 355 | #define FIND_EDGE(DST, FROM, NOEND, INC, STEP0, STEP1, LEN) \ |
| | 356 | for (last_y = y = FROM; NOEND; y = y INC) { \ |
| | 357 | if (checkline_edge(ctx, tmpbuf + STEP0 * y, STEP1, LEN, bpp)) { \ |
| | 358 | if (last_y INC == y) { \ |
| | 359 | DST = y; \ |
| | 360 | break; \ |
| | 361 | } else \ |
| | 362 | last_y = y; \ |
| | 363 | } \ |
| | 364 | } \ |
| | 365 | if (!(NOEND)) { \ |
| | 366 | DST = y -(INC); \ |
| | 367 | } |
| 192 | | 368 | |
| | 369 | FIND_EDGE(s->y1, s->y1, y >= 0, -1, inw, bpp, scan_w); |
| | 370 | FIND_EDGE(s->y2, s->y2, y < inh, +1, inw, bpp, scan_w); |
| | 371 | FIND_EDGE(s->x1, s->x1, y >= 0, -1, bpp, inw, scan_h); |
| | 372 | FIND_EDGE(s->x2, s->x2, y < inw, +1, bpp, inw, scan_h); |
| | 373 | |
| | 374 | // queue bboxes |
| | 375 | bboff = (s->frame_nb - 1) % s->window_size; |
| | 376 | s->bboxes[0][bboff] = s->x1; |
| | 377 | s->bboxes[1][bboff] = s->x2; |
| | 378 | s->bboxes[2][bboff] = s->y1; |
| | 379 | s->bboxes[3][bboff] = s->y2; |
| | 380 | |
| | 381 | // sort queue |
| | 382 | bboff = FFMIN(s->frame_nb, s->window_size); |
| | 383 | AV_QSORT(s->bboxes[0], bboff, int, comp); |
| | 384 | AV_QSORT(s->bboxes[1], bboff, int, comp); |
| | 385 | AV_QSORT(s->bboxes[2], bboff, int, comp); |
| | 386 | AV_QSORT(s->bboxes[3], bboff, int, comp); |
| | 387 | |
| | 388 | // return median of window_size elems |
| | 389 | s->x1 = s->bboxes[0][bboff/2]; |
| | 390 | s->x2 = s->bboxes[1][bboff/2]; |
| | 391 | s->y1 = s->bboxes[2][bboff/2]; |
| | 392 | s->y2 = s->bboxes[3][bboff/2]; |
| | 393 | } |
| | 394 | } |
| 193 | | 395 | |
| 194 | // round x and y (up), important for yuv colorspaces | 396 | // round x and y (up), important for yuv colorspaces |
| 195 | // make sure they stay rounded! | 397 | // make sure they stay rounded! |
| @@ -243,6 +445,12 @@ static const AVOption cropdetect_options[] = { · 47 unmodified lines |
| 243 | { "skip", "Number of initial frames to skip", OFFSET(skip), AV_OPT_TYPE_INT, { .i64 = 2 }, 0, INT_MAX, FLAGS }, | 445 | { "skip", "Number of initial frames to skip", OFFSET(skip), AV_OPT_TYPE_INT, { .i64 = 2 }, 0, INT_MAX, FLAGS }, |
| 244 | { "reset_count", "Recalculate the crop area after this many frames",OFFSET(reset_count),AV_OPT_TYPE_INT,{ .i64 = 0 }, 0, INT_MAX, FLAGS }, | 446 | { "reset_count", "Recalculate the crop area after this many frames",OFFSET(reset_count),AV_OPT_TYPE_INT,{ .i64 = 0 }, 0, INT_MAX, FLAGS }, |
| 245 | { "max_outliers", "Threshold count of outliers", OFFSET(max_outliers),AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, FLAGS }, | 447 | { "max_outliers", "Threshold count of outliers", OFFSET(max_outliers),AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, FLAGS }, |
| | 448 | { "mode", "set mode", OFFSET(mode), AV_OPT_TYPE_INT, {.i64=MODE_BLACK}, 0, MODE_NB-1, FLAGS, "mode" }, |
| | 449 | { "black", "detect black pixels surrounding the video", 0, AV_OPT_TYPE_CONST, {.i64=MODE_BLACK}, INT_MIN, INT_MAX, FLAGS, "mode" }, |
| | 450 | { "mvedges", "detect motion and edged surrounding the video", 0, AV_OPT_TYPE_CONST, {.i64=MODE_MV_EDGES}, INT_MIN, INT_MAX, FLAGS, "mode" }, |
| | 451 | { "high", "Set high threshold for edge detection", OFFSET(high), AV_OPT_TYPE_FLOAT, {.dbl=25/255.}, 0, 1, FLAGS }, |
| | 452 | { "low", "Set low threshold for edge detection", OFFSET(low), AV_OPT_TYPE_FLOAT, {.dbl=15/255.}, 0, 1, FLAGS }, |
| | 453 | { "mv_threshold", "motion vector threshold when estimating video window size", OFFSET(mv_threshold), AV_OPT_TYPE_INT, {.i64=8}, 0, 100, FLAGS}, |
| 246 | { NULL } | 454 | { NULL } |
| 247 | }; | 455 | }; |
| 248 | | 456 | |
| @@ -270,6 +478,7 @@ const AVFilter ff_vf_cropdetect = { · 21 unmodified lines |
| 270 | .priv_size = sizeof(CropDetectContext), | 478 | .priv_size = sizeof(CropDetectContext), |
| 271 | .priv_class = &cropdetect_class, | 479 | .priv_class = &cropdetect_class, |
| 272 | .init = init, | 480 | .init = init, |
| | 481 | .uninit = uninit, |
| 273 | FILTER_INPUTS(avfilter_vf_cropdetect_inputs), | 482 | FILTER_INPUTS(avfilter_vf_cropdetect_inputs), |
| 274 | FILTER_OUTPUTS(avfilter_vf_cropdetect_outputs), | 483 | FILTER_OUTPUTS(avfilter_vf_cropdetect_outputs), |
| 275 | FILTER_PIXFMTS_ARRAY(pix_fmts), | 484 | FILTER_PIXFMTS_ARRAY(pix_fmts), |