| @@ -0,0 +1,2202 @@ |
| | 1 | /* |
| | 2 | * This file is part of FFmpeg. |
| | 3 | * |
| | 4 | * FFmpeg is free software; you can redistribute it and/or |
| | 5 | * modify it under the terms of the GNU Lesser General Public |
| | 6 | * License as published by the Free Software Foundation; either |
| | 7 | * version 2.1 of the License, or (at your option) any later version. |
| | 8 | * |
| | 9 | * FFmpeg is distributed in the hope that it will be useful, |
| | 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| | 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| | 12 | * Lesser General Public License for more details. |
| | 13 | * |
| | 14 | * You should have received a copy of the GNU Lesser General Public |
| | 15 | * License along with FFmpeg; if not, write to the Free Software |
| | 16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
| | 17 | * |
| | 18 | * Copyright (C) 2000-2008, Intel Corporation, all rights reserved. |
| | 19 | * Copyright (C) 2009, Willow Garage Inc., all rights reserved. |
| | 20 | * Copyright (C) 2013, OpenCV Foundation, all rights reserved. |
| | 21 | * Third party copyrights are property of their respective owners. |
| | 22 | * |
| | 23 | * Redistribution and use in source and binary forms, with or without modification, |
| | 24 | * are permitted provided that the following conditions are met: |
| | 25 | * |
| | 26 | * * Redistribution's of source code must retain the above copyright notice, |
| | 27 | * this list of conditions and the following disclaimer. |
| | 28 | * |
| | 29 | * * Redistribution's in binary form must reproduce the above copyright notice, |
| | 30 | * this list of conditions and the following disclaimer in the documentation |
| | 31 | * and/or other materials provided with the distribution. |
| | 32 | * |
| | 33 | * * The name of the copyright holders may not be used to endorse or promote products |
| | 34 | * derived from this software without specific prior written permission. |
| | 35 | * |
| | 36 | * This software is provided by the copyright holders and contributors "as is" and |
| | 37 | * any express or implied warranties, including, but not limited to, the implied |
| | 38 | * warranties of merchantability and fitness for a particular purpose are disclaimed. |
| | 39 | * In no event shall the Intel Corporation or contributors be liable for any direct, |
| | 40 | * indirect, incidental, special, exemplary, or consequential damages |
| | 41 | * (including, but not limited to, procurement of substitute goods or services; |
| | 42 | * loss of use, data, or profits; or business interruption) however caused |
| | 43 | * and on any theory of liability, whether in contract, strict liability, |
| | 44 | * or tort (including negligence or otherwise) arising in any way out of |
| | 45 | * the use of this software, even if advised of the possibility of such damage. |
| | 46 | */ |
| | 47 | |
| | 48 | #include <stdbool.h> |
| | 49 | #include <float.h> |
| | 50 | #include <libavutil/lfg.h> |
| | 51 | #include "libavutil/opt.h" |
| | 52 | #include "libavutil/imgutils.h" |
| | 53 | #include "libavutil/mem.h" |
| | 54 | #include "libavutil/fifo.h" |
| | 55 | #include "libavutil/common.h" |
| | 56 | #include "libavutil/avassert.h" |
| | 57 | #include "libavutil/pixfmt.h" |
| | 58 | #include "avfilter.h" |
| | 59 | #include "framequeue.h" |
| | 60 | #include "filters.h" |
| | 61 | #include "transform.h" |
| | 62 | #include "formats.h" |
| | 63 | #include "internal.h" |
| | 64 | #include "opencl.h" |
| | 65 | #include "opencl_source.h" |
| | 66 | #include "video.h" |
| | 67 | |
| | 68 | /* |
| | 69 | This filter matches feature points between frames (dealing with outliers) and then |
| | 70 | uses the matches to estimate an affine transform between frames. This transform is |
| | 71 | decomposed into various values (translation, scale, rotation) and the values are |
| | 72 | summed relative to the start of the video to obtain on absolute camera position |
| | 73 | for each frame. This "camera path" is then smoothed via a gaussian filter, resulting |
| | 74 | in a new path that is turned back into an affine transform and applied to each |
| | 75 | frame to render it. |
| | 76 | |
| | 77 | High-level overview: |
| | 78 | |
| | 79 | All of the work to extract motion data from frames occurs in queue_frame. Motion data |
| | 80 | is buffered in a smoothing window, so queue_frame simply computes the absolute camera |
| | 81 | positions and places them in ringbuffers. |
| | 82 | |
| | 83 | filter_frame is responsible for looking at the absolute camera positions currently |
| | 84 | in the ringbuffers, applying the gaussian filter, and then transforming the frames. |
| | 85 | */ |
| | 86 | |
| | 87 | // Number of bits for BRIEF descriptors |
| | 88 | #define BREIFN 512 |
| | 89 | // Size of the patch from which a BRIEF descriptor is extracted |
| | 90 | // This is the size used in OpenCV |
| | 91 | #define BRIEF_PATCH_SIZE 31 |
| | 92 | #define BRIEF_PATCH_SIZE_HALF (BRIEF_PATCH_SIZE / 2) |
| | 93 | |
| | 94 | #define MATCHES_CONTIG_SIZE 2000 |
| | 95 | |
| | 96 | #define ROUNDED_UP_DIV(a, b) ((a + (b - 1)) / b) |
| | 97 | |
| | 98 | typedef struct PointPair { |
| | 99 | // Previous frame |
| | 100 | cl_float2 p1; |
| | 101 | // Current frame |
| | 102 | cl_float2 p2; |
| | 103 | } PointPair; |
| | 104 | |
| | 105 | typedef struct MotionVector { |
| | 106 | PointPair p; |
| | 107 | // Used to mark vectors as potential outliers |
| | 108 | cl_int should_consider; |
| | 109 | } MotionVector; |
| | 110 | |
| | 111 | // Denotes the indices for the different types of motion in the ringbuffers array |
| | 112 | enum RingbufferIndices { |
| | 113 | RingbufX, |
| | 114 | RingbufY, |
| | 115 | RingbufRot, |
| | 116 | RingbufScaleX, |
| | 117 | RingbufScaleY, |
| | 118 | |
| | 119 | // Should always be last |
| | 120 | RingbufCount |
| | 121 | }; |
| | 122 | |
| | 123 | // Struct that holds data for drawing point match debug data |
| | 124 | typedef struct DebugMatches { |
| | 125 | MotionVector *matches; |
| | 126 | // The points used to calculate the affine transform for a frame |
| | 127 | MotionVector model_matches[3]; |
| | 128 | |
| | 129 | int num_matches; |
| | 130 | // For cases where we couldn't calculate a model |
| | 131 | int num_model_matches; |
| | 132 | } DebugMatches; |
| | 133 | |
| | 134 | // Groups together the ringbuffers that store absolute distortion / position values |
| | 135 | // for each frame |
| | 136 | typedef struct AbsoluteFrameMotion { |
| | 137 | // Array with the various ringbuffers, indexed via the RingbufferIndices enum |
| | 138 | AVFifoBuffer *ringbuffers[RingbufCount]; |
| | 139 | |
| | 140 | // Offset to get to the current frame being processed |
| | 141 | // (not in bytes) |
| | 142 | int curr_frame_offset; |
| | 143 | // Keeps track of where the start and end of contiguous motion data is (to |
| | 144 | // deal with cases where no motion data is found between two frames) |
| | 145 | int data_start_offset; |
| | 146 | int data_end_offset; |
| | 147 | |
| | 148 | AVFifoBuffer *debug_matches; |
| | 149 | } AbsoluteFrameMotion; |
| | 150 | |
| | 151 | // Takes care of freeing the arrays within the DebugMatches inside of the |
| | 152 | // debug_matches ringbuffer and then freeing the buffer itself. |
| | 153 | static void free_debug_matches(AbsoluteFrameMotion *afm) { |
| | 154 | DebugMatches dm; |
| | 155 | |
| | 156 | if (!afm->debug_matches) { |
| | 157 | return; |
| | 158 | } |
| | 159 | |
| | 160 | while (av_fifo_size(afm->debug_matches) > 0) { |
| | 161 | av_fifo_generic_read( |
| | 162 | afm->debug_matches, |
| | 163 | &dm, |
| | 164 | sizeof(DebugMatches), |
| | 165 | NULL |
| | 166 | ); |
| | 167 | |
| | 168 | av_freep(&dm.matches); |
| | 169 | } |
| | 170 | |
| | 171 | av_fifo_freep(&afm->debug_matches); |
| | 172 | } |
| | 173 | |
| | 174 | // Stores the translation, scale, rotation, and skew deltas between two frames |
| | 175 | typedef struct FrameDelta { |
| | 176 | cl_float2 translation; |
| | 177 | float rotation; |
| | 178 | cl_float2 scale; |
| | 179 | cl_float2 skew; |
| | 180 | } FrameDelta; |
| | 181 | |
| | 182 | typedef struct SimilarityMatrix { |
| | 183 | // The 2x3 similarity matrix |
| | 184 | double matrix[6]; |
| | 185 | } SimilarityMatrix; |
| | 186 | |
| | 187 | typedef struct CropInfo { |
| | 188 | // The top left corner of the bounding box for the crop |
| | 189 | cl_float2 top_left; |
| | 190 | // The bottom right corner of the bounding box for the crop |
| | 191 | cl_float2 bottom_right; |
| | 192 | } CropInfo; |
| | 193 | |
| | 194 | // Returned from function that determines start and end values for iteration |
| | 195 | // around the current frame in a ringbuffer |
| | 196 | typedef struct IterIndices { |
| | 197 | int start; |
| | 198 | int end; |
| | 199 | } IterIndices; |
| | 200 | |
| | 201 | typedef struct DeshakeOpenCLContext { |
| | 202 | OpenCLFilterContext ocf; |
| | 203 | // Whether or not the above `OpenCLFilterContext` has been initialized |
| | 204 | int initialized; |
| | 205 | |
| | 206 | // These variables are used in the activate callback |
| | 207 | int64_t duration; |
| | 208 | bool eof; |
| | 209 | |
| | 210 | // State for random number generation |
| | 211 | AVLFG alfg; |
| | 212 | |
| | 213 | // FIFO frame queue used to buffer future frames for processing |
| | 214 | FFFrameQueue fq; |
| | 215 | // Ringbuffers for frame positions |
| | 216 | AbsoluteFrameMotion abs_motion; |
| | 217 | |
| | 218 | // The number of frames' motion to consider before and after the frame we are |
| | 219 | // smoothing |
| | 220 | int smooth_window; |
| | 221 | // The number of the frame we are currently processing |
| | 222 | int curr_frame; |
| | 223 | |
| | 224 | // Stores a 1d array of normalised gaussian kernel values for convolution |
| | 225 | float *gauss_kernel; |
| | 226 | |
| | 227 | // Buffer for error values used in RANSAC code |
| | 228 | float *ransac_err; |
| | 229 | |
| | 230 | // Information regarding how to crop the smoothed luminance (or RGB) planes |
| | 231 | CropInfo crop_y; |
| | 232 | // Information regarding how to crop the smoothed chroma planes |
| | 233 | CropInfo crop_uv; |
| | 234 | |
| | 235 | // Whether or not we are processing YUV input (as oppposed to RGB) |
| | 236 | bool is_yuv; |
| | 237 | // The underlying format of the hardware surfaces |
| | 238 | int sw_format; |
| | 239 | |
| | 240 | // Buffer to copy `matches` into for the CPU to work with |
| | 241 | MotionVector *matches_host; |
| | 242 | MotionVector *matches_contig_host; |
| | 243 | |
| | 244 | MotionVector *inliers; |
| | 245 | |
| | 246 | cl_command_queue command_queue; |
| | 247 | cl_kernel kernel_grayscale; |
| | 248 | cl_kernel kernel_harris_response; |
| | 249 | cl_kernel kernel_refine_features; |
| | 250 | cl_kernel kernel_brief_descriptors; |
| | 251 | cl_kernel kernel_match_descriptors; |
| | 252 | cl_kernel kernel_transform; |
| | 253 | cl_kernel kernel_crop_upscale; |
| | 254 | |
| | 255 | // Stores a frame converted to grayscale |
| | 256 | cl_mem grayscale; |
| | 257 | // Stores the harris response for a frame (measure of "cornerness" for each pixel) |
| | 258 | cl_mem harris_buf; |
| | 259 | |
| | 260 | // Detected features after non-maximum suppression and sub-pixel refinement |
| | 261 | cl_mem refined_features; |
| | 262 | // Saved from the previous frame |
| | 263 | cl_mem prev_refined_features; |
| | 264 | |
| | 265 | // BRIEF sampling pattern that is randomly initialized |
| | 266 | cl_mem brief_pattern; |
| | 267 | // Feature point descriptors for the current frame |
| | 268 | cl_mem descriptors; |
| | 269 | // Feature point descriptors for the previous frame |
| | 270 | cl_mem prev_descriptors; |
| | 271 | // Vectors between points in current and previous frame |
| | 272 | cl_mem matches; |
| | 273 | cl_mem matches_contig; |
| | 274 | // Holds the matrix to transform luminance (or RGB) with |
| | 275 | cl_mem transform_y; |
| | 276 | // Holds the matrix to transform chroma with |
| | 277 | cl_mem transform_uv; |
| | 278 | |
| | 279 | // Configurable options |
| | 280 | |
| | 281 | int tripod_mode; |
| | 282 | int debug_on; |
| | 283 | int should_crop; |
| | 284 | |
| | 285 | // Whether or not feature points should be refined at a sub-pixel level |
| | 286 | cl_int refine_features; |
| | 287 | // If the user sets a value other than the default, 0, this percentage is |
| | 288 | // translated into a sigma value ranging from 0.5 to 40.0 |
| | 289 | float smooth_percent; |
| | 290 | // This number is multiplied by the video frame rate to determine the size |
| | 291 | // of the smooth window |
| | 292 | float smooth_window_multiplier; |
| | 293 | |
| | 294 | // Debug stuff |
| | 295 | |
| | 296 | cl_kernel kernel_draw_debug_info; |
| | 297 | cl_mem debug_matches; |
| | 298 | cl_mem debug_model_matches; |
| | 299 | |
| | 300 | // These store the total time spent executing the different kernels in nanoseconds |
| | 301 | unsigned long long grayscale_time; |
| | 302 | unsigned long long harris_response_time; |
| | 303 | unsigned long long refine_features_time; |
| | 304 | unsigned long long brief_descriptors_time; |
| | 305 | unsigned long long match_descriptors_time; |
| | 306 | unsigned long long transform_time; |
| | 307 | unsigned long long crop_upscale_time; |
| | 308 | |
| | 309 | // Time spent copying matched features from the device to the host |
| | 310 | unsigned long long read_buf_time; |
| | 311 | } DeshakeOpenCLContext; |
| | 312 | |
| | 313 | // Returns a random uniformly-distributed number in [low, high] |
| | 314 | static int rand_in(int low, int high, AVLFG *alfg) { |
| | 315 | return (av_lfg_get(alfg) % (high - low)) + low; |
| | 316 | } |
| | 317 | |
| | 318 | // Returns the average execution time for an event given the total time and the |
| | 319 | // number of frames processed. |
| | 320 | static double averaged_event_time_ms(unsigned long long total_time, int num_frames) { |
| | 321 | return (double)total_time / (double)num_frames / 1000000.0; |
| | 322 | } |
| | 323 | |
| | 324 | // The following code is loosely ported from OpenCV |
| | 325 | |
| | 326 | // Estimates affine transform from 3 point pairs |
| | 327 | // model is a 2x3 matrix: |
| | 328 | // a b c |
| | 329 | // d e f |
| | 330 | static void run_estimate_kernel(const MotionVector *point_pairs, double *model) |
| | 331 | { |
| | 332 | // src points |
| | 333 | double x1 = point_pairs[0].p.p1.s[0]; |
| | 334 | double y1 = point_pairs[0].p.p1.s[1]; |
| | 335 | double x2 = point_pairs[1].p.p1.s[0]; |
| | 336 | double y2 = point_pairs[1].p.p1.s[1]; |
| | 337 | double x3 = point_pairs[2].p.p1.s[0]; |
| | 338 | double y3 = point_pairs[2].p.p1.s[1]; |
| | 339 | |
| | 340 | // dest points |
| | 341 | double X1 = point_pairs[0].p.p2.s[0]; |
| | 342 | double Y1 = point_pairs[0].p.p2.s[1]; |
| | 343 | double X2 = point_pairs[1].p.p2.s[0]; |
| | 344 | double Y2 = point_pairs[1].p.p2.s[1]; |
| | 345 | double X3 = point_pairs[2].p.p2.s[0]; |
| | 346 | double Y3 = point_pairs[2].p.p2.s[1]; |
| | 347 | |
| | 348 | double d = 1.0 / ( x1*(y2-y3) + x2*(y3-y1) + x3*(y1-y2) ); |
| | 349 | |
| | 350 | model[0] = d * ( X1*(y2-y3) + X2*(y3-y1) + X3*(y1-y2) ); |
| | 351 | model[1] = d * ( X1*(x3-x2) + X2*(x1-x3) + X3*(x2-x1) ); |
| | 352 | model[2] = d * ( X1*(x2*y3 - x3*y2) + X2*(x3*y1 - x1*y3) + X3*(x1*y2 - x2*y1) ); |
| | 353 | |
| | 354 | model[3] = d * ( Y1*(y2-y3) + Y2*(y3-y1) + Y3*(y1-y2) ); |
| | 355 | model[4] = d * ( Y1*(x3-x2) + Y2*(x1-x3) + Y3*(x2-x1) ); |
| | 356 | model[5] = d * ( Y1*(x2*y3 - x3*y2) + Y2*(x3*y1 - x1*y3) + Y3*(x1*y2 - x2*y1) ); |
| | 357 | } |
| | 358 | |
| | 359 | // Checks that the 3 points in the given array are not collinear |
| | 360 | static bool points_not_collinear(const cl_float2 **points) |
| | 361 | { |
| | 362 | int j, k, i = 2; |
| | 363 | |
| | 364 | for (j = 0; j < i; j++) { |
| | 365 | double dx1 = points[j]->s[0] - points[i]->s[0]; |
| | 366 | double dy1 = points[j]->s[1] - points[i]->s[1]; |
| | 367 | |
| | 368 | for (k = 0; k < j; k++) { |
| | 369 | double dx2 = points[k]->s[0] - points[i]->s[0]; |
| | 370 | double dy2 = points[k]->s[1] - points[i]->s[1]; |
| | 371 | |
| | 372 | // Assuming a 3840 x 2160 video with a point at (0, 0) and one at |
| | 373 | // (3839, 2159), this prevents a third point from being within roughly |
| | 374 | // 0.5 of a pixel of the line connecting the two on both axes |
| | 375 | if (fabs(dx2*dy1 - dy2*dx1) <= 1.0) { |
| | 376 | return false; |
| | 377 | } |
| | 378 | } |
| | 379 | } |
| | 380 | |
| | 381 | return true; |
| | 382 | } |
| | 383 | |
| | 384 | // Checks a subset of 3 point pairs to make sure that the points are not collinear |
| | 385 | // and not too close to each other |
| | 386 | static bool check_subset(const MotionVector *pairs_subset) |
| | 387 | { |
| | 388 | const cl_float2 *prev_points[] = { |
| | 389 | &pairs_subset[0].p.p1, |
| | 390 | &pairs_subset[1].p.p1, |
| | 391 | &pairs_subset[2].p.p1 |
| | 392 | }; |
| | 393 | |
| | 394 | const cl_float2 *curr_points[] = { |
| | 395 | &pairs_subset[0].p.p2, |
| | 396 | &pairs_subset[1].p.p2, |
| | 397 | &pairs_subset[2].p.p2 |
| | 398 | }; |
| | 399 | |
| | 400 | return points_not_collinear(prev_points) && points_not_collinear(curr_points); |
| | 401 | } |
| | 402 | |
| | 403 | // Selects a random subset of 3 points from point_pairs and places them in pairs_subset |
| | 404 | static bool get_subset( |
| | 405 | AVLFG *alfg, |
| | 406 | const MotionVector *point_pairs, |
| | 407 | const int num_point_pairs, |
| | 408 | MotionVector *pairs_subset, |
| | 409 | int max_attempts |
| | 410 | ) { |
| | 411 | int idx[3]; |
| | 412 | int i = 0, j, iters = 0; |
| | 413 | |
| | 414 | for (; iters < max_attempts; iters++) { |
| | 415 | for (i = 0; i < 3 && iters < max_attempts;) { |
| | 416 | int idx_i = 0; |
| | 417 | |
| | 418 | for (;;) { |
| | 419 | idx_i = idx[i] = rand_in(0, num_point_pairs, alfg); |
| | 420 | |
| | 421 | for (j = 0; j < i; j++) { |
| | 422 | if (idx_i == idx[j]) { |
| | 423 | break; |
| | 424 | } |
| | 425 | } |
| | 426 | |
| | 427 | if (j == i) { |
| | 428 | break; |
| | 429 | } |
| | 430 | } |
| | 431 | |
| | 432 | pairs_subset[i] = point_pairs[idx[i]]; |
| | 433 | i++; |
| | 434 | } |
| | 435 | |
| | 436 | if (i == 3 && !check_subset(pairs_subset)) { |
| | 437 | continue; |
| | 438 | } |
| | 439 | break; |
| | 440 | } |
| | 441 | |
| | 442 | return i == 3 && iters < max_attempts; |
| | 443 | } |
| | 444 | |
| | 445 | // Computes the error for each of the given points based on the given model. |
| | 446 | static void compute_error( |
| | 447 | const MotionVector *point_pairs, |
| | 448 | const int num_point_pairs, |
| | 449 | const double *model, |
| | 450 | float *err |
| | 451 | ) { |
| | 452 | double F0 = model[0], F1 = model[1], F2 = model[2]; |
| | 453 | double F3 = model[3], F4 = model[4], F5 = model[5]; |
| | 454 | |
| | 455 | for (int i = 0; i < num_point_pairs; i++) { |
| | 456 | const cl_float2 *f = &point_pairs[i].p.p1; |
| | 457 | const cl_float2 *t = &point_pairs[i].p.p2; |
| | 458 | |
| | 459 | double a = F0*f->s[0] + F1*f->s[1] + F2 - t->s[0]; |
| | 460 | double b = F3*f->s[0] + F4*f->s[1] + F5 - t->s[1]; |
| | 461 | |
| | 462 | err[i] = a*a + b*b; |
| | 463 | } |
| | 464 | } |
| | 465 | |
| | 466 | // Determines which of the given point matches are inliers for the given model |
| | 467 | // based on the specified threshold. |
| | 468 | // |
| | 469 | // err must be an array of num_point_pairs length |
| | 470 | static int find_inliers( |
| | 471 | MotionVector *point_pairs, |
| | 472 | const int num_point_pairs, |
| | 473 | const double *model, |
| | 474 | float *err, |
| | 475 | double thresh |
| | 476 | ) { |
| | 477 | float t = (float)(thresh * thresh); |
| | 478 | int i, n = num_point_pairs, num_inliers = 0; |
| | 479 | |
| | 480 | compute_error(point_pairs, num_point_pairs, model, err); |
| | 481 | |
| | 482 | for (i = 0; i < n; i++) { |
| | 483 | if (err[i] <= t) { |
| | 484 | // This is an inlier |
| | 485 | point_pairs[i].should_consider = true; |
| | 486 | num_inliers += 1; |
| | 487 | } else { |
| | 488 | point_pairs[i].should_consider = false; |
| | 489 | } |
| | 490 | } |
| | 491 | |
| | 492 | return num_inliers; |
| | 493 | } |
| | 494 | |
| | 495 | // Determines the number of iterations required to achieve the desired confidence level. |
| | 496 | // |
| | 497 | // The equation used to determine the number of iterations to do is: |
| | 498 | // 1 - confidence = (1 - inlier_probability^num_points)^num_iters |
| | 499 | // |
| | 500 | // Solving for num_iters: |
| | 501 | // |
| | 502 | // num_iters = log(1 - confidence) / log(1 - inlier_probability^num_points) |
| | 503 | // |
| | 504 | // A more in-depth explanation can be found at https://en.wikipedia.org/wiki/Random_sample_consensus |
| | 505 | // under the 'Parameters' heading |
| | 506 | static int ransac_update_num_iters(double confidence, double num_outliers, int max_iters) |
| | 507 | { |
| | 508 | double num, denom; |
| | 509 | |
| | 510 | confidence = av_clipd(confidence, 0.0, 1.0); |
| | 511 | num_outliers = av_clipd(num_outliers, 0.0, 1.0); |
| | 512 | |
| | 513 | // avoid inf's & nan's |
| | 514 | num = FFMAX(1.0 - confidence, DBL_MIN); |
| | 515 | denom = 1.0 - pow(1.0 - num_outliers, 3); |
| | 516 | if (denom < DBL_MIN) { |
| | 517 | return 0; |
| | 518 | } |
| | 519 | |
| | 520 | num = log(num); |
| | 521 | denom = log(denom); |
| | 522 | |
| | 523 | return denom >= 0 || -num >= max_iters * (-denom) ? max_iters : (int)round(num / denom); |
| | 524 | } |
| | 525 | |
| | 526 | // Estimates an affine transform between the given pairs of points using RANdom |
| | 527 | // SAmple Consensus |
| | 528 | static bool estimate_affine_2d( |
| | 529 | DeshakeOpenCLContext *deshake_ctx, |
| | 530 | MotionVector *point_pairs, |
| | 531 | DebugMatches *debug_matches, |
| | 532 | const int num_point_pairs, |
| | 533 | double *model_out, |
| | 534 | const double threshold, |
| | 535 | const int max_iters, |
| | 536 | const double confidence |
| | 537 | ) { |
| | 538 | bool result = false; |
| | 539 | double best_model[6], model[6]; |
| | 540 | MotionVector pairs_subset[3], best_pairs[3]; |
| | 541 | |
| | 542 | int iter, niters = FFMAX(max_iters, 1); |
| | 543 | int good_count, max_good_count = 0; |
| | 544 | |
| | 545 | // We need at least 3 points to build a model from |
| | 546 | if (num_point_pairs < 3) { |
| | 547 | return false; |
| | 548 | } else if (num_point_pairs == 3) { |
| | 549 | // There are only 3 points, so RANSAC doesn't apply here |
| | 550 | run_estimate_kernel(point_pairs, model_out); |
| | 551 | |
| | 552 | for (int i = 0; i < 3; ++i) { |
| | 553 | point_pairs[i].should_consider = true; |
| | 554 | } |
| | 555 | |
| | 556 | return true; |
| | 557 | } |
| | 558 | |
| | 559 | for (iter = 0; iter < niters; ++iter) { |
| | 560 | bool found = get_subset(&deshake_ctx->alfg, point_pairs, num_point_pairs, pairs_subset, 10000); |
| | 561 | |
| | 562 | if (!found) { |
| | 563 | if (iter == 0) { |
| | 564 | return false; |
| | 565 | } |
| | 566 | |
| | 567 | break; |
| | 568 | } |
| | 569 | |
| | 570 | run_estimate_kernel(pairs_subset, model); |
| | 571 | good_count = find_inliers(point_pairs, num_point_pairs, model, deshake_ctx->ransac_err, threshold); |
| | 572 | |
| | 573 | if (good_count > FFMAX(max_good_count, 2)) { |
| | 574 | for (int mi = 0; mi < 6; ++mi) { |
| | 575 | best_model[mi] = model[mi]; |
| | 576 | } |
| | 577 | |
| | 578 | for (int pi = 0; pi < 3; pi++) { |
| | 579 | best_pairs[pi] = pairs_subset[pi]; |
| | 580 | } |
| | 581 | |
| | 582 | max_good_count = good_count; |
| | 583 | niters = ransac_update_num_iters( |
| | 584 | confidence, |
| | 585 | (double)(num_point_pairs - good_count) / num_point_pairs, |
| | 586 | niters |
| | 587 | ); |
| | 588 | } |
| | 589 | } |
| | 590 | |
| | 591 | if (max_good_count > 0) { |
| | 592 | for (int mi = 0; mi < 6; ++mi) { |
| | 593 | model_out[mi] = best_model[mi]; |
| | 594 | } |
| | 595 | |
| | 596 | for (int pi = 0; pi < 3; ++pi) { |
| | 597 | debug_matches->model_matches[pi] = best_pairs[pi]; |
| | 598 | } |
| | 599 | debug_matches->num_model_matches = 3; |
| | 600 | |
| | 601 | // Find the inliers again for the best model for debugging |
| | 602 | find_inliers(point_pairs, num_point_pairs, best_model, deshake_ctx->ransac_err, threshold); |
| | 603 | result = true; |
| | 604 | } |
| | 605 | |
| | 606 | return result; |
| | 607 | } |
| | 608 | |
| | 609 | // "Wiggles" the first point in best_pairs around a tiny bit in order to decrease the |
| | 610 | // total error |
| | 611 | static void optimize_model( |
| | 612 | DeshakeOpenCLContext *deshake_ctx, |
| | 613 | MotionVector *best_pairs, |
| | 614 | MotionVector *inliers, |
| | 615 | const int num_inliers, |
| | 616 | float best_err, |
| | 617 | double *model_out |
| | 618 | ) { |
| | 619 | float move_x_val = 0.01; |
| | 620 | float move_y_val = 0.01; |
| | 621 | bool move_x = true; |
| | 622 | float old_move_x_val = 0; |
| | 623 | double model[6]; |
| | 624 | int last_changed = 0; |
| | 625 | |
| | 626 | for (int iters = 0; iters < 200; iters++) { |
| | 627 | float total_err = 0; |
| | 628 | |
| | 629 | if (move_x) { |
| | 630 | best_pairs[0].p.p2.s[0] += move_x_val; |
| | 631 | } else { |
| | 632 | best_pairs[0].p.p2.s[0] += move_y_val; |
| | 633 | } |
| | 634 | |
| | 635 | run_estimate_kernel(best_pairs, model); |
| | 636 | compute_error(inliers, num_inliers, model, deshake_ctx->ransac_err); |
| | 637 | |
| | 638 | for (int j = 0; j < num_inliers; j++) { |
| | 639 | total_err += deshake_ctx->ransac_err[j]; |
| | 640 | } |
| | 641 | |
| | 642 | if (total_err < best_err) { |
| | 643 | for (int mi = 0; mi < 6; ++mi) { |
| | 644 | model_out[mi] = model[mi]; |
| | 645 | } |
| | 646 | |
| | 647 | best_err = total_err; |
| | 648 | last_changed = iters; |
| | 649 | } else { |
| | 650 | // Undo the change |
| | 651 | if (move_x) { |
| | 652 | best_pairs[0].p.p2.s[0] -= move_x_val; |
| | 653 | } else { |
| | 654 | best_pairs[0].p.p2.s[0] -= move_y_val; |
| | 655 | } |
| | 656 | |
| | 657 | if (iters - last_changed > 4) { |
| | 658 | // We've already improved the model as much as we can |
| | 659 | break; |
| | 660 | } |
| | 661 | |
| | 662 | old_move_x_val = move_x_val; |
| | 663 | |
| | 664 | if (move_x) { |
| | 665 | move_x_val *= -1; |
| | 666 | } else { |
| | 667 | move_y_val *= -1; |
| | 668 | } |
| | 669 | |
| | 670 | if (old_move_x_val < 0) { |
| | 671 | move_x = false; |
| | 672 | } else { |
| | 673 | move_x = true; |
| | 674 | } |
| | 675 | } |
| | 676 | } |
| | 677 | } |
| | 678 | |
| | 679 | // Uses a process similar to that of RANSAC to find a transform that minimizes |
| | 680 | // the total error for a set of point matches determined to be inliers |
| | 681 | // |
| | 682 | // (Pick random subsets, compute model, find total error, iterate until error |
| | 683 | // is minimized.) |
| | 684 | static bool minimize_error( |
| | 685 | DeshakeOpenCLContext *deshake_ctx, |
| | 686 | MotionVector *inliers, |
| | 687 | DebugMatches *debug_matches, |
| | 688 | const int num_inliers, |
| | 689 | double *model_out, |
| | 690 | const int max_iters |
| | 691 | ) { |
| | 692 | bool result = false; |
| | 693 | float best_err = FLT_MAX; |
| | 694 | double best_model[6], model[6]; |
| | 695 | MotionVector pairs_subset[3], best_pairs[3]; |
| | 696 | |
| | 697 | for (int i = 0; i < max_iters; i++) { |
| | 698 | float total_err = 0; |
| | 699 | bool found = get_subset(&deshake_ctx->alfg, inliers, num_inliers, pairs_subset, 10000); |
| | 700 | |
| | 701 | if (!found) { |
| | 702 | if (i == 0) { |
| | 703 | return false; |
| | 704 | } |
| | 705 | |
| | 706 | break; |
| | 707 | } |
| | 708 | |
| | 709 | run_estimate_kernel(pairs_subset, model); |
| | 710 | compute_error(inliers, num_inliers, model, deshake_ctx->ransac_err); |
| | 711 | |
| | 712 | for (int j = 0; j < num_inliers; j++) { |
| | 713 | total_err += deshake_ctx->ransac_err[j]; |
| | 714 | } |
| | 715 | |
| | 716 | if (total_err < best_err) { |
| | 717 | for (int mi = 0; mi < 6; ++mi) { |
| | 718 | best_model[mi] = model[mi]; |
| | 719 | } |
| | 720 | |
| | 721 | for (int pi = 0; pi < 3; pi++) { |
| | 722 | best_pairs[pi] = pairs_subset[pi]; |
| | 723 | } |
| | 724 | |
| | 725 | best_err = total_err; |
| | 726 | } |
| | 727 | } |
| | 728 | |
| | 729 | for (int mi = 0; mi < 6; ++mi) { |
| | 730 | model_out[mi] = best_model[mi]; |
| | 731 | } |
| | 732 | |
| | 733 | for (int pi = 0; pi < 3; ++pi) { |
| | 734 | debug_matches->model_matches[pi] = best_pairs[pi]; |
| | 735 | } |
| | 736 | debug_matches->num_model_matches = 3; |
| | 737 | result = true; |
| | 738 | |
| | 739 | optimize_model(deshake_ctx, best_pairs, inliers, num_inliers, best_err, model_out); |
| | 740 | return result; |
| | 741 | } |
| | 742 | |
| | 743 | // End code from OpenCV |
| | 744 | |
| | 745 | // Decomposes a similarity matrix into translation, rotation, scale, and skew |
| | 746 | // |
| | 747 | // See http://frederic-wang.fr/decomposition-of-2d-transform-matrices.html |
| | 748 | static FrameDelta decompose_transform(double *model) |
| | 749 | { |
| | 750 | FrameDelta ret; |
| | 751 | |
| | 752 | double a = model[0]; |
| | 753 | double c = model[1]; |
| | 754 | double e = model[2]; |
| | 755 | double b = model[3]; |
| | 756 | double d = model[4]; |
| | 757 | double f = model[5]; |
| | 758 | double delta = a * d - b * c; |
| | 759 | |
| | 760 | ret.translation.s[0] = e; |
| | 761 | ret.translation.s[1] = f; |
| | 762 | |
| | 763 | // This is the QR method |
| | 764 | if (a != 0 || b != 0) { |
| | 765 | double r = hypot(a, b); |
| | 766 | |
| | 767 | ret.rotation = FFSIGN(b) * acos(a / r); |
| | 768 | ret.scale.s[0] = r; |
| | 769 | ret.scale.s[1] = delta / r; |
| | 770 | ret.skew.s[0] = atan((a * c + b * d) / (r * r)); |
| | 771 | ret.skew.s[1] = 0; |
| | 772 | } else if (c != 0 || d != 0) { |
| | 773 | double s = sqrt(c * c + d * d); |
| | 774 | |
| | 775 | ret.rotation = M_PI / 2 - FFSIGN(d) * acos(-c / s); |
| | 776 | ret.scale.s[0] = delta / s; |
| | 777 | ret.scale.s[1] = s; |
| | 778 | ret.skew.s[0] = 0; |
| | 779 | ret.skew.s[1] = atan((a * c + b * d) / (s * s)); |
| | 780 | } // otherwise there is only translation |
| | 781 | |
| | 782 | return ret; |
| | 783 | } |
| | 784 | |
| | 785 | // Move valid vectors from the 2d buffer into a 1d buffer where they are contiguous |
| | 786 | static int make_vectors_contig( |
| | 787 | DeshakeOpenCLContext *deshake_ctx, |
| | 788 | int size_y, |
| | 789 | int size_x |
| | 790 | ) { |
| | 791 | int num_vectors = 0; |
| | 792 | |
| | 793 | for (int i = 0; i < size_y; ++i) { |
| | 794 | for (int j = 0; j < size_x; ++j) { |
| | 795 | MotionVector v = deshake_ctx->matches_host[j + i * size_x]; |
| | 796 | |
| | 797 | if (v.should_consider) { |
| | 798 | deshake_ctx->matches_contig_host[num_vectors] = v; |
| | 799 | ++num_vectors; |
| | 800 | } |
| | 801 | |
| | 802 | // Make sure we do not exceed the amount of space we allocated for these vectors |
| | 803 | if (num_vectors == MATCHES_CONTIG_SIZE - 1) { |
| | 804 | return num_vectors; |
| | 805 | } |
| | 806 | } |
| | 807 | } |
| | 808 | return num_vectors; |
| | 809 | } |
| | 810 | |
| | 811 | // Returns the gaussian kernel value for the given x coordinate and sigma value |
| | 812 | static float gaussian_for(int x, float sigma) { |
| | 813 | return 1.0f / expf(((float)x * (float)x) / (2.0f * sigma * sigma)); |
| | 814 | } |
| | 815 | |
| | 816 | // Makes a normalized gaussian kernel of the given length for the given sigma |
| | 817 | // and places it in gauss_kernel |
| | 818 | static void make_gauss_kernel(float *gauss_kernel, float length, float sigma) |
| | 819 | { |
| | 820 | float gauss_sum = 0; |
| | 821 | int window_half = length / 2; |
| | 822 | |
| | 823 | for (int i = 0; i < length; ++i) { |
| | 824 | float val = gaussian_for(i - window_half, sigma); |
| | 825 | |
| | 826 | gauss_sum += val; |
| | 827 | gauss_kernel[i] = val; |
| | 828 | } |
| | 829 | |
| | 830 | // Normalize the gaussian values |
| | 831 | for (int i = 0; i < length; ++i) { |
| | 832 | gauss_kernel[i] /= gauss_sum; |
| | 833 | } |
| | 834 | } |
| | 835 | |
| | 836 | // Returns indices to start and end iteration at in order to iterate over a window |
| | 837 | // of length size centered at the current frame in a ringbuffer |
| | 838 | // |
| | 839 | // Always returns numbers that result in a window of length size, even if that |
| | 840 | // means specifying negative indices or indices past the end of the values in the |
| | 841 | // ringbuffers. Make sure you clip indices appropriately within your loop. |
| | 842 | static IterIndices start_end_for(DeshakeOpenCLContext *deshake_ctx, int length) { |
| | 843 | IterIndices indices; |
| | 844 | |
| | 845 | indices.start = deshake_ctx->abs_motion.curr_frame_offset - (length / 2); |
| | 846 | indices.end = deshake_ctx->abs_motion.curr_frame_offset + (length / 2) + (length % 2); |
| | 847 | |
| | 848 | return indices; |
| | 849 | } |
| | 850 | |
| | 851 | // Sets val to the value in the given ringbuffer at the given offset, taking care of |
| | 852 | // clipping the offset into the appropriate range |
| | 853 | static void ringbuf_float_at( |
| | 854 | DeshakeOpenCLContext *deshake_ctx, |
| | 855 | AVFifoBuffer *values, |
| | 856 | float *val, |
| | 857 | int offset |
| | 858 | ) { |
| | 859 | int clip_start, clip_end, offset_clipped; |
| | 860 | if (deshake_ctx->abs_motion.data_end_offset != -1) { |
| | 861 | clip_end = deshake_ctx->abs_motion.data_end_offset; |
| | 862 | } else { |
| | 863 | // This expression represents the last valid index in the buffer, |
| | 864 | // which we use repeatedly at the end of the video. |
| | 865 | clip_end = deshake_ctx->smooth_window - (av_fifo_space(values) / sizeof(float)) - 1; |
| | 866 | } |
| | 867 | |
| | 868 | if (deshake_ctx->abs_motion.data_start_offset != -1) { |
| | 869 | clip_start = deshake_ctx->abs_motion.data_start_offset; |
| | 870 | } else { |
| | 871 | // Negative indices will occur at the start of the video, and we want |
| | 872 | // them to be clipped to 0 in order to repeatedly use the position of |
| | 873 | // the first frame. |
| | 874 | clip_start = 0; |
| | 875 | } |
| | 876 | |
| | 877 | offset_clipped = av_clip( |
| | 878 | offset, |
| | 879 | clip_start, |
| | 880 | clip_end |
| | 881 | ); |
| | 882 | |
| | 883 | av_fifo_generic_peek_at( |
| | 884 | values, |
| | 885 | val, |
| | 886 | offset_clipped * sizeof(float), |
| | 887 | sizeof(float), |
| | 888 | NULL |
| | 889 | ); |
| | 890 | } |
| | 891 | |
| | 892 | // Returns smoothed current frame value of the given buffer of floats based on the |
| | 893 | // given Gaussian kernel and its length (also the window length, centered around the |
| | 894 | // current frame) and the "maximum value" of the motion. |
| | 895 | // |
| | 896 | // This "maximum value" should be the width / height of the image in the case of |
| | 897 | // translation and an empirically chosen constant for rotation / scale. |
| | 898 | // |
| | 899 | // The sigma chosen to generate the final gaussian kernel with used to smooth the |
| | 900 | // camera path is either hardcoded (set by user, deshake_ctx->smooth_percent) or |
| | 901 | // adaptively chosen. |
| | 902 | static float smooth( |
| | 903 | DeshakeOpenCLContext *deshake_ctx, |
| | 904 | float *gauss_kernel, |
| | 905 | int length, |
| | 906 | float max_val, |
| | 907 | AVFifoBuffer *values |
| | 908 | ) { |
| | 909 | float new_large_s = 0, new_small_s = 0, new_best = 0, old, diff_between, |
| | 910 | percent_of_max, inverted_percent; |
| | 911 | IterIndices indices = start_end_for(deshake_ctx, length); |
| | 912 | float large_sigma = 40.0f; |
| | 913 | float small_sigma = 2.0f; |
| | 914 | float best_sigma; |
| | 915 | |
| | 916 | if (deshake_ctx->smooth_percent) { |
| | 917 | best_sigma = (large_sigma - 0.5f) * deshake_ctx->smooth_percent + 0.5f; |
| | 918 | } else { |
| | 919 | // Strategy to adaptively smooth trajectory: |
| | 920 | // |
| | 921 | // 1. Smooth path with large and small sigma values |
| | 922 | // 2. Take the absolute value of the difference between them |
| | 923 | // 3. Get a percentage by putting the difference over the "max value" |
| | 924 | // 4, Invert the percentage |
| | 925 | // 5. Calculate a new sigma value weighted towards the larger sigma value |
| | 926 | // 6. Determine final smoothed trajectory value using that sigma |
| | 927 | |
| | 928 | make_gauss_kernel(gauss_kernel, length, large_sigma); |
| | 929 | for (int i = indices.start, j = 0; i < indices.end; ++i, ++j) { |
| | 930 | ringbuf_float_at(deshake_ctx, values, &old, i); |
| | 931 | new_large_s += old * gauss_kernel[j]; |
| | 932 | } |
| | 933 | |
| | 934 | make_gauss_kernel(gauss_kernel, length, small_sigma); |
| | 935 | for (int i = indices.start, j = 0; i < indices.end; ++i, ++j) { |
| | 936 | ringbuf_float_at(deshake_ctx, values, &old, i); |
| | 937 | new_small_s += old * gauss_kernel[j]; |
| | 938 | } |
| | 939 | |
| | 940 | diff_between = fabsf(new_large_s - new_small_s); |
| | 941 | percent_of_max = diff_between / max_val; |
| | 942 | inverted_percent = 1 - percent_of_max; |
| | 943 | best_sigma = large_sigma * powf(inverted_percent, 40); |
| | 944 | } |
| | 945 | |
| | 946 | make_gauss_kernel(gauss_kernel, length, best_sigma); |
| | 947 | for (int i = indices.start, j = 0; i < indices.end; ++i, ++j) { |
| | 948 | ringbuf_float_at(deshake_ctx, values, &old, i); |
| | 949 | new_best += old * gauss_kernel[j]; |
| | 950 | } |
| | 951 | |
| | 952 | return new_best; |
| | 953 | } |
| | 954 | |
| | 955 | // Returns the position of the given point after the transform is applied |
| | 956 | static cl_float2 transformed_point(float x, float y, float *transform) { |
| | 957 | cl_float2 ret; |
| | 958 | |
| | 959 | ret.s[0] = x * transform[0] + y * transform[1] + transform[2]; |
| | 960 | ret.s[1] = x * transform[3] + y * transform[4] + transform[5]; |
| | 961 | |
| | 962 | return ret; |
| | 963 | } |
| | 964 | |
| | 965 | // Creates an affine transform that scales from the center of a frame |
| | 966 | static void transform_center_scale( |
| | 967 | float x_shift, |
| | 968 | float y_shift, |
| | 969 | float angle, |
| | 970 | float scale_x, |
| | 971 | float scale_y, |
| | 972 | float center_w, |
| | 973 | float center_h, |
| | 974 | float *matrix |
| | 975 | ) { |
| | 976 | cl_float2 center_s; |
| | 977 | float center_s_w, center_s_h; |
| | 978 | |
| | 979 | ff_get_matrix( |
| | 980 | 0, |
| | 981 | 0, |
| | 982 | 0, |
| | 983 | scale_x, |
| | 984 | scale_y, |
| | 985 | matrix |
| | 986 | ); |
| | 987 | |
| | 988 | center_s = transformed_point(center_w, center_h, matrix); |
| | 989 | center_s_w = center_w - center_s.s[0]; |
| | 990 | center_s_h = center_h - center_s.s[1]; |
| | 991 | |
| | 992 | ff_get_matrix( |
| | 993 | x_shift + center_s_w, |
| | 994 | y_shift + center_s_h, |
| | 995 | angle, |
| | 996 | scale_x, |
| | 997 | scale_y, |
| | 998 | matrix |
| | 999 | ); |
| | 1000 | } |
| | 1001 | |
| | 1002 | // Determines the crop necessary to eliminate black borders from a smoothed frame |
| | 1003 | // and updates target crop accordingly |
| | 1004 | static void update_needed_crop( |
| | 1005 | CropInfo* crop, |
| | 1006 | float *transform, |
| | 1007 | float frame_width, |
| | 1008 | float frame_height |
| | 1009 | ) { |
| | 1010 | float new_width, new_height, adjusted_width, adjusted_height, adjusted_x, adjusted_y; |
| | 1011 | |
| | 1012 | cl_float2 top_left = transformed_point(0, 0, transform); |
| | 1013 | cl_float2 top_right = transformed_point(frame_width, 0, transform); |
| | 1014 | cl_float2 bottom_left = transformed_point(0, frame_height, transform); |
| | 1015 | cl_float2 bottom_right = transformed_point(frame_width, frame_height, transform); |
| | 1016 | float ar_h = frame_height / frame_width; |
| | 1017 | float ar_w = frame_width / frame_height; |
| | 1018 | |
| | 1019 | if (crop->bottom_right.s[0] == 0) { |
| | 1020 | // The crop hasn't been set to the original size of the plane |
| | 1021 | crop->bottom_right.s[0] = frame_width; |
| | 1022 | crop->bottom_right.s[1] = frame_height; |
| | 1023 | } |
| | 1024 | |
| | 1025 | crop->top_left.s[0] = FFMAX3( |
| | 1026 | crop->top_left.s[0], |
| | 1027 | top_left.s[0], |
| | 1028 | bottom_left.s[0] |
| | 1029 | ); |
| | 1030 | |
| | 1031 | crop->top_left.s[1] = FFMAX3( |
| | 1032 | crop->top_left.s[1], |
| | 1033 | top_left.s[1], |
| | 1034 | top_right.s[1] |
| | 1035 | ); |
| | 1036 | |
| | 1037 | crop->bottom_right.s[0] = FFMIN3( |
| | 1038 | crop->bottom_right.s[0], |
| | 1039 | bottom_right.s[0], |
| | 1040 | top_right.s[0] |
| | 1041 | ); |
| | 1042 | |
| | 1043 | crop->bottom_right.s[1] = FFMIN3( |
| | 1044 | crop->bottom_right.s[1], |
| | 1045 | bottom_right.s[1], |
| | 1046 | bottom_left.s[1] |
| | 1047 | ); |
| | 1048 | |
| | 1049 | // Make sure our potentially new bounding box has the same aspect ratio |
| | 1050 | new_height = crop->bottom_right.s[1] - crop->top_left.s[1]; |
| | 1051 | new_width = crop->bottom_right.s[0] - crop->top_left.s[0]; |
| | 1052 | |
| | 1053 | adjusted_width = new_height * ar_w; |
| | 1054 | adjusted_x = crop->bottom_right.s[0] - adjusted_width; |
| | 1055 | |
| | 1056 | if (adjusted_x >= crop->top_left.s[0]) { |
| | 1057 | crop->top_left.s[0] = adjusted_x; |
| | 1058 | } else { |
| | 1059 | adjusted_height = new_width * ar_h; |
| | 1060 | adjusted_y = crop->bottom_right.s[1] - adjusted_height; |
| | 1061 | crop->top_left.s[1] = adjusted_y; |
| | 1062 | } |
| | 1063 | } |
| | 1064 | |
| | 1065 | static av_cold void deshake_opencl_uninit(AVFilterContext *avctx) |
| | 1066 | { |
| | 1067 | DeshakeOpenCLContext *ctx = avctx->priv; |
| | 1068 | cl_int cle; |
| | 1069 | |
| | 1070 | for (int i = 0; i < RingbufCount; i++) |
| | 1071 | av_fifo_freep(&ctx->abs_motion.ringbuffers[i]); |
| | 1072 | |
| | 1073 | if (ctx->debug_on) |
| | 1074 | free_debug_matches(&ctx->abs_motion); |
| | 1075 | |
| | 1076 | if (ctx->gauss_kernel) |
| | 1077 | av_freep(&ctx->gauss_kernel); |
| | 1078 | |
| | 1079 | if (ctx->ransac_err) |
| | 1080 | av_freep(&ctx->ransac_err); |
| | 1081 | |
| | 1082 | if (ctx->matches_host) |
| | 1083 | av_freep(&ctx->matches_host); |
| | 1084 | |
| | 1085 | if (ctx->matches_contig_host) |
| | 1086 | av_freep(&ctx->matches_contig_host); |
| | 1087 | |
| | 1088 | if (ctx->inliers) |
| | 1089 | av_freep(&ctx->inliers); |
| | 1090 | |
| | 1091 | ff_framequeue_free(&ctx->fq); |
| | 1092 | |
| | 1093 | CL_RELEASE_KERNEL(ctx->kernel_grayscale); |
| | 1094 | CL_RELEASE_KERNEL(ctx->kernel_harris_response); |
| | 1095 | CL_RELEASE_KERNEL(ctx->kernel_refine_features); |
| | 1096 | CL_RELEASE_KERNEL(ctx->kernel_brief_descriptors); |
| | 1097 | CL_RELEASE_KERNEL(ctx->kernel_match_descriptors); |
| | 1098 | CL_RELEASE_KERNEL(ctx->kernel_crop_upscale); |
| | 1099 | if (ctx->debug_on) |
| | 1100 | CL_RELEASE_KERNEL(ctx->kernel_draw_debug_info); |
| | 1101 | |
| | 1102 | CL_RELEASE_QUEUE(ctx->command_queue); |
| | 1103 | |
| | 1104 | if (!ctx->is_yuv) |
| | 1105 | CL_RELEASE_MEMORY(ctx->grayscale); |
| | 1106 | CL_RELEASE_MEMORY(ctx->harris_buf); |
| | 1107 | CL_RELEASE_MEMORY(ctx->refined_features); |
| | 1108 | CL_RELEASE_MEMORY(ctx->prev_refined_features); |
| | 1109 | CL_RELEASE_MEMORY(ctx->brief_pattern); |
| | 1110 | CL_RELEASE_MEMORY(ctx->descriptors); |
| | 1111 | CL_RELEASE_MEMORY(ctx->prev_descriptors); |
| | 1112 | CL_RELEASE_MEMORY(ctx->matches); |
| | 1113 | CL_RELEASE_MEMORY(ctx->matches_contig); |
| | 1114 | CL_RELEASE_MEMORY(ctx->transform_y); |
| | 1115 | CL_RELEASE_MEMORY(ctx->transform_uv); |
| | 1116 | if (ctx->debug_on) { |
| | 1117 | CL_RELEASE_MEMORY(ctx->debug_matches); |
| | 1118 | CL_RELEASE_MEMORY(ctx->debug_model_matches); |
| | 1119 | } |
| | 1120 | |
| | 1121 | ff_opencl_filter_uninit(avctx); |
| | 1122 | } |
| | 1123 | |
| | 1124 | static int deshake_opencl_init(AVFilterContext *avctx) |
| | 1125 | { |
| | 1126 | DeshakeOpenCLContext *ctx = avctx->priv; |
| | 1127 | AVFilterLink *outlink = avctx->outputs[0]; |
| | 1128 | AVFilterLink *inlink = avctx->inputs[0]; |
| | 1129 | // Pointer to the host-side pattern buffer to be initialized and then copied |
| | 1130 | // to the GPU |
| | 1131 | PointPair *pattern_host; |
| | 1132 | cl_int cle; |
| | 1133 | int err; |
| | 1134 | cl_ulong8 zeroed_ulong8; |
| | 1135 | FFFrameQueueGlobal fqg; |
| | 1136 | cl_image_format grayscale_format; |
| | 1137 | cl_image_desc grayscale_desc; |
| | 1138 | cl_command_queue_properties queue_props; |
| | 1139 | |
| | 1140 | const enum AVPixelFormat disallowed_formats[14] = { |
| | 1141 | AV_PIX_FMT_GBRP, |
| | 1142 | AV_PIX_FMT_GBRP9BE, |
| | 1143 | AV_PIX_FMT_GBRP9LE, |
| | 1144 | AV_PIX_FMT_GBRP10BE, |
| | 1145 | AV_PIX_FMT_GBRP10LE, |
| | 1146 | AV_PIX_FMT_GBRP16BE, |
| | 1147 | AV_PIX_FMT_GBRP16LE, |
| | 1148 | AV_PIX_FMT_GBRAP, |
| | 1149 | AV_PIX_FMT_GBRAP16BE, |
| | 1150 | AV_PIX_FMT_GBRAP16LE, |
| | 1151 | AV_PIX_FMT_GBRAP12BE, |
| | 1152 | AV_PIX_FMT_GBRAP12LE, |
| | 1153 | AV_PIX_FMT_GBRAP10BE, |
| | 1154 | AV_PIX_FMT_GBRAP10LE |
| | 1155 | }; |
| | 1156 | |
| | 1157 | // Number of elements for an array |
| | 1158 | const int image_grid_32 = ROUNDED_UP_DIV(outlink->h, 32) * ROUNDED_UP_DIV(outlink->w, 32); |
| | 1159 | |
| | 1160 | const int descriptor_buf_size = image_grid_32 * (BREIFN / 8); |
| | 1161 | const int features_buf_size = image_grid_32 * sizeof(cl_float2); |
| | 1162 | |
| | 1163 | const AVHWFramesContext *hw_frames_ctx = (AVHWFramesContext*)inlink->hw_frames_ctx->data; |
| | 1164 | const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(hw_frames_ctx->sw_format); |
| | 1165 | |
| | 1166 | av_assert0(hw_frames_ctx); |
| | 1167 | av_assert0(desc); |
| | 1168 | |
| | 1169 | ff_framequeue_global_init(&fqg); |
| | 1170 | ff_framequeue_init(&ctx->fq, &fqg); |
| | 1171 | ctx->eof = false; |
| | 1172 | ctx->smooth_window = (int)(av_q2d(avctx->inputs[0]->frame_rate) * ctx->smooth_window_multiplier); |
| | 1173 | ctx->curr_frame = 0; |
| | 1174 | |
| | 1175 | memset(&zeroed_ulong8, 0, sizeof(cl_ulong8)); |
| | 1176 | |
| | 1177 | ctx->gauss_kernel = av_malloc_array(ctx->smooth_window, sizeof(float)); |
| | 1178 | if (!ctx->gauss_kernel) { |
| | 1179 | err = AVERROR(ENOMEM); |
| | 1180 | goto fail; |
| | 1181 | } |
| | 1182 | |
| | 1183 | ctx->ransac_err = av_malloc_array(MATCHES_CONTIG_SIZE, sizeof(float)); |
| | 1184 | if (!ctx->ransac_err) { |
| | 1185 | err = AVERROR(ENOMEM); |
| | 1186 | goto fail; |
| | 1187 | } |
| | 1188 | |
| | 1189 | for (int i = 0; i < RingbufCount; i++) { |
| | 1190 | ctx->abs_motion.ringbuffers[i] = av_fifo_alloc_array( |
| | 1191 | ctx->smooth_window, |
| | 1192 | sizeof(float) |
| | 1193 | ); |
| | 1194 | |
| | 1195 | if (!ctx->abs_motion.ringbuffers[i]) { |
| | 1196 | err = AVERROR(ENOMEM); |
| | 1197 | goto fail; |
| | 1198 | } |
| | 1199 | } |
| | 1200 | |
| | 1201 | if (ctx->debug_on) { |
| | 1202 | ctx->abs_motion.debug_matches = av_fifo_alloc_array( |
| | 1203 | ctx->smooth_window / 2, |
| | 1204 | sizeof(DebugMatches) |
| | 1205 | ); |
| | 1206 | |
| | 1207 | if (!ctx->abs_motion.debug_matches) { |
| | 1208 | err = AVERROR(ENOMEM); |
| | 1209 | goto fail; |
| | 1210 | } |
| | 1211 | } |
| | 1212 | |
| | 1213 | ctx->abs_motion.curr_frame_offset = 0; |
| | 1214 | ctx->abs_motion.data_start_offset = -1; |
| | 1215 | ctx->abs_motion.data_end_offset = -1; |
| | 1216 | |
| | 1217 | pattern_host = av_malloc_array(BREIFN, sizeof(PointPair)); |
| | 1218 | if (!pattern_host) { |
| | 1219 | err = AVERROR(ENOMEM); |
| | 1220 | goto fail; |
| | 1221 | } |
| | 1222 | |
| | 1223 | ctx->matches_host = av_malloc_array(image_grid_32, sizeof(MotionVector)); |
| | 1224 | if (!ctx->matches_host) { |
| | 1225 | err = AVERROR(ENOMEM); |
| | 1226 | goto fail; |
| | 1227 | } |
| | 1228 | |
| | 1229 | ctx->matches_contig_host = av_malloc_array(MATCHES_CONTIG_SIZE, sizeof(MotionVector)); |
| | 1230 | if (!ctx->matches_contig_host) { |
| | 1231 | err = AVERROR(ENOMEM); |
| | 1232 | goto fail; |
| | 1233 | } |
| | 1234 | |
| | 1235 | ctx->inliers = av_malloc_array(MATCHES_CONTIG_SIZE, sizeof(MotionVector)); |
| | 1236 | if (!ctx->inliers) { |
| | 1237 | err = AVERROR(ENOMEM); |
| | 1238 | goto fail; |
| | 1239 | } |
| | 1240 | |
| | 1241 | // Initializing the patch pattern for building BREIF descriptors with |
| | 1242 | av_lfg_init(&ctx->alfg, 234342424); |
| | 1243 | for (int i = 0; i < BREIFN; ++i) { |
| | 1244 | PointPair pair; |
| | 1245 | |
| | 1246 | for (int j = 0; j < 2; ++j) { |
| | 1247 | pair.p1.s[j] = rand_in(-BRIEF_PATCH_SIZE_HALF, BRIEF_PATCH_SIZE_HALF + 1, &ctx->alfg); |
| | 1248 | pair.p2.s[j] = rand_in(-BRIEF_PATCH_SIZE_HALF, BRIEF_PATCH_SIZE_HALF + 1, &ctx->alfg); |
| | 1249 | } |
| | 1250 | |
| | 1251 | pattern_host[i] = pair; |
| | 1252 | } |
| | 1253 | |
| | 1254 | for (int i = 0; i < 14; i++) { |
| | 1255 | if (ctx->sw_format == disallowed_formats[i]) { |
| | 1256 | av_log(avctx, AV_LOG_ERROR, "unsupported format in deshake_opencl.\n"); |
| | 1257 | err = AVERROR(ENOSYS); |
| | 1258 | goto fail; |
| | 1259 | } |
| | 1260 | } |
| | 1261 | |
| | 1262 | if (desc->flags & AV_PIX_FMT_FLAG_RGB) { |
| | 1263 | ctx->is_yuv = false; |
| | 1264 | } else { |
| | 1265 | ctx->is_yuv = true; |
| | 1266 | } |
| | 1267 | ctx->sw_format = hw_frames_ctx->sw_format; |
| | 1268 | |
| | 1269 | err = ff_opencl_filter_load_program(avctx, &ff_opencl_source_deshake, 1); |
| | 1270 | if (err < 0) |
| | 1271 | goto fail; |
| | 1272 | |
| | 1273 | if (ctx->debug_on) { |
| | 1274 | queue_props = CL_QUEUE_PROFILING_ENABLE; |
| | 1275 | } else { |
| | 1276 | queue_props = 0; |
| | 1277 | } |
| | 1278 | ctx->command_queue = clCreateCommandQueue( |
| | 1279 | ctx->ocf.hwctx->context, |
| | 1280 | ctx->ocf.hwctx->device_id, |
| | 1281 | queue_props, |
| | 1282 | &cle |
| | 1283 | ); |
| | 1284 | CL_FAIL_ON_ERROR(AVERROR(EIO), "Failed to create OpenCL command queue %d.\n", cle); |
| | 1285 | |
| | 1286 | CL_CREATE_KERNEL(ctx, grayscale); |
| | 1287 | CL_CREATE_KERNEL(ctx, harris_response); |
| | 1288 | CL_CREATE_KERNEL(ctx, refine_features); |
| | 1289 | CL_CREATE_KERNEL(ctx, brief_descriptors); |
| | 1290 | CL_CREATE_KERNEL(ctx, match_descriptors); |
| | 1291 | CL_CREATE_KERNEL(ctx, transform); |
| | 1292 | CL_CREATE_KERNEL(ctx, crop_upscale); |
| | 1293 | if (ctx->debug_on) |
| | 1294 | CL_CREATE_KERNEL(ctx, draw_debug_info); |
| | 1295 | |
| | 1296 | if (!ctx->is_yuv) { |
| | 1297 | grayscale_format.image_channel_order = CL_R; |
| | 1298 | grayscale_format.image_channel_data_type = CL_FLOAT; |
| | 1299 | |
| | 1300 | grayscale_desc = (cl_image_desc) { |
| | 1301 | .image_type = CL_MEM_OBJECT_IMAGE2D, |
| | 1302 | .image_width = outlink->w, |
| | 1303 | .image_height = outlink->h, |
| | 1304 | .image_depth = 0, |
| | 1305 | .image_array_size = 0, |
| | 1306 | .image_row_pitch = 0, |
| | 1307 | .image_slice_pitch = 0, |
| | 1308 | .num_mip_levels = 0, |
| | 1309 | .num_samples = 0, |
| | 1310 | .buffer = NULL, |
| | 1311 | }; |
| | 1312 | |
| | 1313 | ctx->grayscale = clCreateImage( |
| | 1314 | ctx->ocf.hwctx->context, |
| | 1315 | 0, |
| | 1316 | &grayscale_format, |
| | 1317 | &grayscale_desc, |
| | 1318 | NULL, |
| | 1319 | &cle |
| | 1320 | ); |
| | 1321 | CL_FAIL_ON_ERROR(AVERROR(EIO), "Failed to create grayscale image: %d.\n", cle); |
| | 1322 | } |
| | 1323 | |
| | 1324 | CL_CREATE_BUFFER(ctx, harris_buf, outlink->h * outlink->w * sizeof(float)); |
| | 1325 | CL_CREATE_BUFFER(ctx, refined_features, features_buf_size); |
| | 1326 | CL_CREATE_BUFFER(ctx, prev_refined_features, features_buf_size); |
| | 1327 | CL_CREATE_BUFFER_FLAGS( |
| | 1328 | ctx, |
| | 1329 | brief_pattern, |
| | 1330 | CL_MEM_READ_WRITE | CL_MEM_COPY_HOST_PTR, |
| | 1331 | BREIFN * sizeof(PointPair), |
| | 1332 | pattern_host |
| | 1333 | ); |
| | 1334 | CL_CREATE_BUFFER(ctx, descriptors, descriptor_buf_size); |
| | 1335 | CL_CREATE_BUFFER(ctx, prev_descriptors, descriptor_buf_size); |
| | 1336 | CL_CREATE_BUFFER(ctx, matches, image_grid_32 * sizeof(MotionVector)); |
| | 1337 | CL_CREATE_BUFFER(ctx, matches_contig, MATCHES_CONTIG_SIZE * sizeof(MotionVector)); |
| | 1338 | CL_CREATE_BUFFER(ctx, transform_y, 9 * sizeof(float)); |
| | 1339 | CL_CREATE_BUFFER(ctx, transform_uv, 9 * sizeof(float)); |
| | 1340 | if (ctx->debug_on) { |
| | 1341 | CL_CREATE_BUFFER(ctx, debug_matches, MATCHES_CONTIG_SIZE * sizeof(MotionVector)); |
| | 1342 | CL_CREATE_BUFFER(ctx, debug_model_matches, 3 * sizeof(MotionVector)); |
| | 1343 | } |
| | 1344 | |
| | 1345 | ctx->initialized = 1; |
| | 1346 | av_freep(&pattern_host); |
| | 1347 | |
| | 1348 | return 0; |
| | 1349 | |
| | 1350 | fail: |
| | 1351 | if (!pattern_host) |
| | 1352 | av_freep(&pattern_host); |
| | 1353 | return err; |
| | 1354 | } |
| | 1355 | |
| | 1356 | // Logs debug information about the transform data |
| | 1357 | static void transform_debug(AVFilterContext *avctx, float *new_vals, float *old_vals, int curr_frame) { |
| | 1358 | av_log(avctx, AV_LOG_VERBOSE, |
| | 1359 | "Frame %d:\n" |
| | 1360 | "\tframe moved from: %f x, %f y\n" |
| | 1361 | "\t to: %f x, %f y\n" |
| | 1362 | "\t rotated from: %f degrees\n" |
| | 1363 | "\t to: %f degrees\n" |
| | 1364 | "\t scaled from: %f x, %f y\n" |
| | 1365 | "\t to: %f x, %f y\n" |
| | 1366 | "\n" |
| | 1367 | "\tframe moved by: %f x, %f y\n" |
| | 1368 | "\t rotated by: %f degrees\n" |
| | 1369 | "\t scaled by: %f x, %f y\n", |
| | 1370 | curr_frame, |
| | 1371 | old_vals[RingbufX], old_vals[RingbufY], |
| | 1372 | new_vals[RingbufX], new_vals[RingbufY], |
| | 1373 | old_vals[RingbufRot] * (180.0 / M_PI), |
| | 1374 | new_vals[RingbufRot] * (180.0 / M_PI), |
| | 1375 | old_vals[RingbufScaleX], old_vals[RingbufScaleY], |
| | 1376 | new_vals[RingbufScaleX], new_vals[RingbufScaleY], |
| | 1377 | old_vals[RingbufX] - new_vals[RingbufX], old_vals[RingbufY] - new_vals[RingbufY], |
| | 1378 | old_vals[RingbufRot] * (180.0 / M_PI) - new_vals[RingbufRot] * (180.0 / M_PI), |
| | 1379 | new_vals[RingbufScaleX] / old_vals[RingbufScaleX], new_vals[RingbufScaleY] / old_vals[RingbufScaleY] |
| | 1380 | ); |
| | 1381 | } |
| | 1382 | |
| | 1383 | // Uses the buffered motion information to determine a transform that smooths the |
| | 1384 | // given frame and applies it |
| | 1385 | static int filter_frame(AVFilterLink *link, AVFrame *input_frame) |
| | 1386 | { |
| | 1387 | AVFilterContext *avctx = link->dst; |
| | 1388 | AVFilterLink *outlink = avctx->outputs[0]; |
| | 1389 | DeshakeOpenCLContext *deshake_ctx = avctx->priv; |
| | 1390 | AVFrame *cropped_frame = NULL, *transformed_frame = NULL; |
| | 1391 | int err; |
| | 1392 | cl_int cle; |
| | 1393 | float new_vals[RingbufCount]; |
| | 1394 | float old_vals[RingbufCount]; |
| | 1395 | // Luma (in the case of YUV) transform, or just the transform in the case of RGB |
| | 1396 | float transform_y[9]; |
| | 1397 | // Chroma transform |
| | 1398 | float transform_uv[9]; |
| | 1399 | // Luma crop transform (or RGB) |
| | 1400 | float transform_crop_y[9]; |
| | 1401 | // Chroma crop transform |
| | 1402 | float transform_crop_uv[9]; |
| | 1403 | float transform_debug_rgb[9]; |
| | 1404 | size_t global_work[2]; |
| | 1405 | int64_t duration; |
| | 1406 | cl_mem src, transformed, dst; |
| | 1407 | cl_mem transforms[3]; |
| | 1408 | CropInfo crops[3]; |
| | 1409 | cl_event transform_event, crop_upscale_event; |
| | 1410 | DebugMatches debug_matches; |
| | 1411 | cl_int num_model_matches; |
| | 1412 | |
| | 1413 | const float center_w = (float)input_frame->width / 2; |
| | 1414 | const float center_h = (float)input_frame->height / 2; |
| | 1415 | |
| | 1416 | const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(deshake_ctx->sw_format); |
| | 1417 | const int chroma_width = AV_CEIL_RSHIFT(input_frame->width, desc->log2_chroma_w); |
| | 1418 | const int chroma_height = AV_CEIL_RSHIFT(input_frame->height, desc->log2_chroma_h); |
| | 1419 | |
| | 1420 | const float center_w_chroma = (float)chroma_width / 2; |
| | 1421 | const float center_h_chroma = (float)chroma_height / 2; |
| | 1422 | |
| | 1423 | const float luma_w_over_chroma_w = ((float)input_frame->width / (float)chroma_width); |
| | 1424 | const float luma_h_over_chroma_h = ((float)input_frame->height / (float)chroma_height); |
| | 1425 | |
| | 1426 | if (deshake_ctx->debug_on) { |
| | 1427 | av_fifo_generic_read( |
| | 1428 | deshake_ctx->abs_motion.debug_matches, |
| | 1429 | &debug_matches, |
| | 1430 | sizeof(DebugMatches), |
| | 1431 | NULL |
| | 1432 | ); |
| | 1433 | } |
| | 1434 | |
| | 1435 | if (input_frame->pkt_duration) { |
| | 1436 | duration = input_frame->pkt_duration; |
| | 1437 | } else { |
| | 1438 | duration = av_rescale_q(1, av_inv_q(outlink->frame_rate), outlink->time_base); |
| | 1439 | } |
| | 1440 | deshake_ctx->duration = input_frame->pts + duration; |
| | 1441 | |
| | 1442 | // Get the absolute transform data for this frame |
| | 1443 | for (int i = 0; i < RingbufCount; i++) { |
| | 1444 | av_fifo_generic_peek_at( |
| | 1445 | deshake_ctx->abs_motion.ringbuffers[i], |
| | 1446 | &old_vals[i], |
| | 1447 | deshake_ctx->abs_motion.curr_frame_offset * sizeof(float), |
| | 1448 | sizeof(float), |
| | 1449 | NULL |
| | 1450 | ); |
| | 1451 | } |
| | 1452 | |
| | 1453 | if (deshake_ctx->tripod_mode) { |
| | 1454 | // If tripod mode is turned on we simply undo all motion relative to the |
| | 1455 | // first frame |
| | 1456 | |
| | 1457 | new_vals[RingbufX] = 0.0f; |
| | 1458 | new_vals[RingbufY] = 0.0f; |
| | 1459 | new_vals[RingbufRot] = 0.0f; |
| | 1460 | new_vals[RingbufScaleX] = 1.0f; |
| | 1461 | new_vals[RingbufScaleY] = 1.0f; |
| | 1462 | } else { |
| | 1463 | // Tripod mode is off and we need to smooth a moving camera |
| | 1464 | |
| | 1465 | new_vals[RingbufX] = smooth( |
| | 1466 | deshake_ctx, |
| | 1467 | deshake_ctx->gauss_kernel, |
| | 1468 | deshake_ctx->smooth_window, |
| | 1469 | input_frame->width, |
| | 1470 | deshake_ctx->abs_motion.ringbuffers[RingbufX] |
| | 1471 | ); |
| | 1472 | new_vals[RingbufY] = smooth( |
| | 1473 | deshake_ctx, |
| | 1474 | deshake_ctx->gauss_kernel, |
| | 1475 | deshake_ctx->smooth_window, |
| | 1476 | input_frame->height, |
| | 1477 | deshake_ctx->abs_motion.ringbuffers[RingbufY] |
| | 1478 | ); |
| | 1479 | new_vals[RingbufRot] = smooth( |
| | 1480 | deshake_ctx, |
| | 1481 | deshake_ctx->gauss_kernel, |
| | 1482 | deshake_ctx->smooth_window, |
| | 1483 | M_PI / 4, |
| | 1484 | deshake_ctx->abs_motion.ringbuffers[RingbufRot] |
| | 1485 | ); |
| | 1486 | new_vals[RingbufScaleX] = smooth( |
| | 1487 | deshake_ctx, |
| | 1488 | deshake_ctx->gauss_kernel, |
| | 1489 | deshake_ctx->smooth_window, |
| | 1490 | 2.0f, |
| | 1491 | deshake_ctx->abs_motion.ringbuffers[RingbufScaleX] |
| | 1492 | ); |
| | 1493 | new_vals[RingbufScaleY] = smooth( |
| | 1494 | deshake_ctx, |
| | 1495 | deshake_ctx->gauss_kernel, |
| | 1496 | deshake_ctx->smooth_window, |
| | 1497 | 2.0f, |
| | 1498 | deshake_ctx->abs_motion.ringbuffers[RingbufScaleY] |
| | 1499 | ); |
| | 1500 | } |
| | 1501 | |
| | 1502 | transform_center_scale( |
| | 1503 | old_vals[RingbufX] - new_vals[RingbufX], |
| | 1504 | old_vals[RingbufY] - new_vals[RingbufY], |
| | 1505 | old_vals[RingbufRot] - new_vals[RingbufRot], |
| | 1506 | new_vals[RingbufScaleX] / old_vals[RingbufScaleX], |
| | 1507 | new_vals[RingbufScaleY] / old_vals[RingbufScaleY], |
| | 1508 | center_w, |
| | 1509 | center_h, |
| | 1510 | transform_y |
| | 1511 | ); |
| | 1512 | |
| | 1513 | transform_center_scale( |
| | 1514 | (old_vals[RingbufX] - new_vals[RingbufX]) / luma_w_over_chroma_w, |
| | 1515 | (old_vals[RingbufY] - new_vals[RingbufY]) / luma_h_over_chroma_h, |
| | 1516 | old_vals[RingbufRot] - new_vals[RingbufRot], |
| | 1517 | new_vals[RingbufScaleX] / old_vals[RingbufScaleX], |
| | 1518 | new_vals[RingbufScaleY] / old_vals[RingbufScaleY], |
| | 1519 | center_w_chroma, |
| | 1520 | center_h_chroma, |
| | 1521 | transform_uv |
| | 1522 | ); |
| | 1523 | |
| | 1524 | CL_BLOCKING_WRITE_BUFFER(deshake_ctx->command_queue, deshake_ctx->transform_y, 9 * sizeof(float), transform_y, NULL); |
| | 1525 | CL_BLOCKING_WRITE_BUFFER(deshake_ctx->command_queue, deshake_ctx->transform_uv, 9 * sizeof(float), transform_uv, NULL); |
| | 1526 | |
| | 1527 | if (deshake_ctx->debug_on) |
| | 1528 | transform_debug(avctx, new_vals, old_vals, deshake_ctx->curr_frame); |
| | 1529 | |
| | 1530 | cropped_frame = ff_get_video_buffer(outlink, outlink->w, outlink->h); |
| | 1531 | if (!cropped_frame) { |
| | 1532 | err = AVERROR(ENOMEM); |
| | 1533 | goto fail; |
| | 1534 | } |
| | 1535 | |
| | 1536 | transformed_frame = ff_get_video_buffer(outlink, outlink->w, outlink->h); |
| | 1537 | if (!transformed_frame) { |
| | 1538 | err = AVERROR(ENOMEM); |
| | 1539 | goto fail; |
| | 1540 | } |
| | 1541 | |
| | 1542 | transforms[0] = deshake_ctx->transform_y; |
| | 1543 | transforms[1] = transforms[2] = deshake_ctx->transform_uv; |
| | 1544 | |
| | 1545 | for (int p = 0; p < FF_ARRAY_ELEMS(transformed_frame->data); p++) { |
| | 1546 | // Transform all of the planes appropriately |
| | 1547 | src = (cl_mem)input_frame->data[p]; |
| | 1548 | transformed = (cl_mem)transformed_frame->data[p]; |
| | 1549 | |
| | 1550 | if (!transformed) |
| | 1551 | break; |
| | 1552 | |
| | 1553 | err = ff_opencl_filter_work_size_from_image(avctx, global_work, input_frame, p, 0); |
| | 1554 | if (err < 0) |
| | 1555 | goto fail; |
| | 1556 | |
| | 1557 | CL_RUN_KERNEL_WITH_ARGS( |
| | 1558 | deshake_ctx->command_queue, |
| | 1559 | deshake_ctx->kernel_transform, |
| | 1560 | global_work, |
| | 1561 | NULL, |
| | 1562 | &transform_event, |
| | 1563 | { sizeof(cl_mem), &src }, |
| | 1564 | { sizeof(cl_mem), &transformed }, |
| | 1565 | { sizeof(cl_mem), &transforms[p] }, |
| | 1566 | ); |
| | 1567 | } |
| | 1568 | |
| | 1569 | if (deshake_ctx->debug_on && !deshake_ctx->is_yuv && debug_matches.num_matches > 0) { |
| | 1570 | CL_BLOCKING_WRITE_BUFFER( |
| | 1571 | deshake_ctx->command_queue, |
| | 1572 | deshake_ctx->debug_matches, |
| | 1573 | debug_matches.num_matches * sizeof(MotionVector), |
| | 1574 | debug_matches.matches, |
| | 1575 | NULL |
| | 1576 | ); |
| | 1577 | |
| | 1578 | CL_BLOCKING_WRITE_BUFFER( |
| | 1579 | deshake_ctx->command_queue, |
| | 1580 | deshake_ctx->debug_model_matches, |
| | 1581 | debug_matches.num_model_matches * sizeof(MotionVector), |
| | 1582 | debug_matches.model_matches, |
| | 1583 | NULL |
| | 1584 | ); |
| | 1585 | |
| | 1586 | num_model_matches = debug_matches.num_model_matches; |
| | 1587 | |
| | 1588 | // Invert the transform |
| | 1589 | transform_center_scale( |
| | 1590 | new_vals[RingbufX] - old_vals[RingbufX], |
| | 1591 | new_vals[RingbufY] - old_vals[RingbufY], |
| | 1592 | new_vals[RingbufRot] - old_vals[RingbufRot], |
| | 1593 | old_vals[RingbufScaleX] / new_vals[RingbufScaleX], |
| | 1594 | old_vals[RingbufScaleY] / new_vals[RingbufScaleY], |
| | 1595 | center_w, |
| | 1596 | center_h, |
| | 1597 | transform_debug_rgb |
| | 1598 | ); |
| | 1599 | |
| | 1600 | CL_BLOCKING_WRITE_BUFFER(deshake_ctx->command_queue, deshake_ctx->transform_y, 9 * sizeof(float), transform_debug_rgb, NULL); |
| | 1601 | |
| | 1602 | transformed = (cl_mem)transformed_frame->data[0]; |
| | 1603 | CL_RUN_KERNEL_WITH_ARGS( |
| | 1604 | deshake_ctx->command_queue, |
| | 1605 | deshake_ctx->kernel_draw_debug_info, |
| | 1606 | (size_t[]){ debug_matches.num_matches }, |
| | 1607 | NULL, |
| | 1608 | NULL, |
| | 1609 | { sizeof(cl_mem), &transformed }, |
| | 1610 | { sizeof(cl_mem), &deshake_ctx->debug_matches }, |
| | 1611 | { sizeof(cl_mem), &deshake_ctx->debug_model_matches }, |
| | 1612 | { sizeof(cl_int), &num_model_matches }, |
| | 1613 | { sizeof(cl_mem), &deshake_ctx->transform_y } |
| | 1614 | ); |
| | 1615 | } |
| | 1616 | |
| | 1617 | if (deshake_ctx->should_crop) { |
| | 1618 | // Generate transforms for cropping |
| | 1619 | transform_center_scale( |
| | 1620 | (old_vals[RingbufX] - new_vals[RingbufX]) / 5, |
| | 1621 | (old_vals[RingbufY] - new_vals[RingbufY]) / 5, |
| | 1622 | (old_vals[RingbufRot] - new_vals[RingbufRot]) / 5, |
| | 1623 | new_vals[RingbufScaleX] / old_vals[RingbufScaleX], |
| | 1624 | new_vals[RingbufScaleY] / old_vals[RingbufScaleY], |
| | 1625 | center_w, |
| | 1626 | center_h, |
| | 1627 | transform_crop_y |
| | 1628 | ); |
| | 1629 | update_needed_crop(&deshake_ctx->crop_y, transform_crop_y, input_frame->width, input_frame->height); |
| | 1630 | |
| | 1631 | transform_center_scale( |
| | 1632 | (old_vals[RingbufX] - new_vals[RingbufX]) / (5 * luma_w_over_chroma_w), |
| | 1633 | (old_vals[RingbufY] - new_vals[RingbufY]) / (5 * luma_h_over_chroma_h), |
| | 1634 | (old_vals[RingbufRot] - new_vals[RingbufRot]) / 5, |
| | 1635 | new_vals[RingbufScaleX] / old_vals[RingbufScaleX], |
| | 1636 | new_vals[RingbufScaleY] / old_vals[RingbufScaleY], |
| | 1637 | center_w_chroma, |
| | 1638 | center_h_chroma, |
| | 1639 | transform_crop_uv |
| | 1640 | ); |
| | 1641 | update_needed_crop(&deshake_ctx->crop_uv, transform_crop_uv, chroma_width, chroma_height); |
| | 1642 | |
| | 1643 | crops[0] = deshake_ctx->crop_y; |
| | 1644 | crops[1] = crops[2] = deshake_ctx->crop_uv; |
| | 1645 | |
| | 1646 | for (int p = 0; p < FF_ARRAY_ELEMS(cropped_frame->data); p++) { |
| | 1647 | // Crop all of the planes appropriately |
| | 1648 | dst = (cl_mem)cropped_frame->data[p]; |
| | 1649 | transformed = (cl_mem)transformed_frame->data[p]; |
| | 1650 | |
| | 1651 | if (!dst) |
| | 1652 | break; |
| | 1653 | |
| | 1654 | err = ff_opencl_filter_work_size_from_image(avctx, global_work, input_frame, p, 0); |
| | 1655 | if (err < 0) |
| | 1656 | goto fail; |
| | 1657 | |
| | 1658 | CL_RUN_KERNEL_WITH_ARGS( |
| | 1659 | deshake_ctx->command_queue, |
| | 1660 | deshake_ctx->kernel_crop_upscale, |
| | 1661 | global_work, |
| | 1662 | NULL, |
| | 1663 | &crop_upscale_event, |
| | 1664 | { sizeof(cl_mem), &transformed }, |
| | 1665 | { sizeof(cl_mem), &dst }, |
| | 1666 | { sizeof(cl_float2), &crops[p].top_left }, |
| | 1667 | { sizeof(cl_float2), &crops[p].bottom_right }, |
| | 1668 | ); |
| | 1669 | } |
| | 1670 | } |
| | 1671 | |
| | 1672 | if (deshake_ctx->curr_frame < deshake_ctx->smooth_window / 2) { |
| | 1673 | // This means we are somewhere at the start of the video. We need to |
| | 1674 | // increment the current frame offset until it reaches the center of |
| | 1675 | // the ringbuffers (as the current frame will be located there for |
| | 1676 | // the rest of the video). |
| | 1677 | // |
| | 1678 | // The end of the video is taken care of by draining motion data |
| | 1679 | // one-by-one out of the buffer, causing the (at that point fixed) |
| | 1680 | // offset to move towards later frames' data. |
| | 1681 | ++deshake_ctx->abs_motion.curr_frame_offset; |
| | 1682 | } |
| | 1683 | |
| | 1684 | if (deshake_ctx->abs_motion.data_end_offset != -1) { |
| | 1685 | // Keep the end offset in sync with the frame it's supposed to be |
| | 1686 | // positioned at |
| | 1687 | --deshake_ctx->abs_motion.data_end_offset; |
| | 1688 | |
| | 1689 | if (deshake_ctx->abs_motion.data_end_offset == deshake_ctx->abs_motion.curr_frame_offset - 1) { |
| | 1690 | // The end offset would be the start of the new video sequence; flip to |
| | 1691 | // start offset |
| | 1692 | deshake_ctx->abs_motion.data_end_offset = -1; |
| | 1693 | deshake_ctx->abs_motion.data_start_offset = deshake_ctx->abs_motion.curr_frame_offset; |
| | 1694 | } |
| | 1695 | } else if (deshake_ctx->abs_motion.data_start_offset != -1) { |
| | 1696 | // Keep the start offset in sync with the frame it's supposed to be |
| | 1697 | // positioned at |
| | 1698 | --deshake_ctx->abs_motion.data_start_offset; |
| | 1699 | } |
| | 1700 | |
| | 1701 | if (deshake_ctx->debug_on) { |
| | 1702 | deshake_ctx->transform_time += ff_opencl_get_event_time(transform_event); |
| | 1703 | if (deshake_ctx->should_crop) { |
| | 1704 | deshake_ctx->crop_upscale_time += ff_opencl_get_event_time(crop_upscale_event); |
| | 1705 | } |
| | 1706 | } |
| | 1707 | |
| | 1708 | ++deshake_ctx->curr_frame; |
| | 1709 | |
| | 1710 | if (deshake_ctx->debug_on) |
| | 1711 | av_freep(&debug_matches.matches); |
| | 1712 | |
| | 1713 | if (deshake_ctx->should_crop) { |
| | 1714 | err = av_frame_copy_props(cropped_frame, input_frame); |
| | 1715 | if (err < 0) |
| | 1716 | goto fail; |
| | 1717 | |
| | 1718 | av_frame_free(&transformed_frame); |
| | 1719 | av_frame_free(&input_frame); |
| | 1720 | return ff_filter_frame(outlink, cropped_frame); |
| | 1721 | |
| | 1722 | } else { |
| | 1723 | err = av_frame_copy_props(transformed_frame, input_frame); |
| | 1724 | if (err < 0) |
| | 1725 | goto fail; |
| | 1726 | |
| | 1727 | av_frame_free(&cropped_frame); |
| | 1728 | av_frame_free(&input_frame); |
| | 1729 | return ff_filter_frame(outlink, transformed_frame); |
| | 1730 | } |
| | 1731 | |
| | 1732 | fail: |
| | 1733 | clFinish(deshake_ctx->command_queue); |
| | 1734 | |
| | 1735 | if (deshake_ctx->debug_on) |
| | 1736 | if (debug_matches.matches) |
| | 1737 | av_freep(&debug_matches.matches); |
| | 1738 | |
| | 1739 | av_frame_free(&input_frame); |
| | 1740 | av_frame_free(&transformed_frame); |
| | 1741 | av_frame_free(&cropped_frame); |
| | 1742 | return err; |
| | 1743 | } |
| | 1744 | |
| | 1745 | // Add the given frame to the frame queue to eventually be processed. |
| | 1746 | // |
| | 1747 | // Also determines the motion from the previous frame and updates the stored |
| | 1748 | // motion information accordingly. |
| | 1749 | static int queue_frame(AVFilterLink *link, AVFrame *input_frame) |
| | 1750 | { |
| | 1751 | AVFilterContext *avctx = link->dst; |
| | 1752 | DeshakeOpenCLContext *deshake_ctx = avctx->priv; |
| | 1753 | int err; |
| | 1754 | int num_vectors; |
| | 1755 | int num_inliers = 0; |
| | 1756 | cl_int cle; |
| | 1757 | FrameDelta relative; |
| | 1758 | SimilarityMatrix model; |
| | 1759 | size_t global_work[2]; |
| | 1760 | size_t harris_global_work[2]; |
| | 1761 | size_t grid_32_global_work[2]; |
| | 1762 | int grid_32_h, grid_32_w; |
| | 1763 | size_t local_work[2]; |
| | 1764 | cl_mem src, temp; |
| | 1765 | float prev_vals[5]; |
| | 1766 | float new_vals[5]; |
| | 1767 | cl_event grayscale_event, harris_response_event, refine_features_event, |
| | 1768 | brief_event, match_descriptors_event, read_buf_event; |
| | 1769 | DebugMatches debug_matches; |
| | 1770 | |
| | 1771 | num_vectors = 0; |
| | 1772 | |
| | 1773 | local_work[0] = 8; |
| | 1774 | local_work[1] = 8; |
| | 1775 | |
| | 1776 | err = ff_opencl_filter_work_size_from_image(avctx, global_work, input_frame, 0, 0); |
| | 1777 | if (err < 0) |
| | 1778 | goto fail; |
| | 1779 | |
| | 1780 | err = ff_opencl_filter_work_size_from_image(avctx, harris_global_work, input_frame, 0, 8); |
| | 1781 | if (err < 0) |
| | 1782 | goto fail; |
| | 1783 | |
| | 1784 | err = ff_opencl_filter_work_size_from_image(avctx, grid_32_global_work, input_frame, 0, 32); |
| | 1785 | if (err < 0) |
| | 1786 | goto fail; |
| | 1787 | |
| | 1788 | // We want a single work-item for each 32x32 block of pixels in the input frame |
| | 1789 | grid_32_global_work[0] /= 32; |
| | 1790 | grid_32_global_work[1] /= 32; |
| | 1791 | |
| | 1792 | grid_32_h = ROUNDED_UP_DIV(input_frame->height, 32); |
| | 1793 | grid_32_w = ROUNDED_UP_DIV(input_frame->width, 32); |
| | 1794 | |
| | 1795 | if (deshake_ctx->is_yuv) { |
| | 1796 | deshake_ctx->grayscale = (cl_mem)input_frame->data[0]; |
| | 1797 | } else { |
| | 1798 | src = (cl_mem)input_frame->data[0]; |
| | 1799 | |
| | 1800 | CL_RUN_KERNEL_WITH_ARGS( |
| | 1801 | deshake_ctx->command_queue, |
| | 1802 | deshake_ctx->kernel_grayscale, |
| | 1803 | global_work, |
| | 1804 | NULL, |
| | 1805 | &grayscale_event, |
| | 1806 | { sizeof(cl_mem), &src }, |
| | 1807 | { sizeof(cl_mem), &deshake_ctx->grayscale } |
| | 1808 | ); |
| | 1809 | } |
| | 1810 | |
| | 1811 | CL_RUN_KERNEL_WITH_ARGS( |
| | 1812 | deshake_ctx->command_queue, |
| | 1813 | deshake_ctx->kernel_harris_response, |
| | 1814 | harris_global_work, |
| | 1815 | local_work, |
| | 1816 | &harris_response_event, |
| | 1817 | { sizeof(cl_mem), &deshake_ctx->grayscale }, |
| | 1818 | { sizeof(cl_mem), &deshake_ctx->harris_buf } |
| | 1819 | ); |
| | 1820 | |
| | 1821 | CL_RUN_KERNEL_WITH_ARGS( |
| | 1822 | deshake_ctx->command_queue, |
| | 1823 | deshake_ctx->kernel_refine_features, |
| | 1824 | grid_32_global_work, |
| | 1825 | NULL, |
| | 1826 | &refine_features_event, |
| | 1827 | { sizeof(cl_mem), &deshake_ctx->grayscale }, |
| | 1828 | { sizeof(cl_mem), &deshake_ctx->harris_buf }, |
| | 1829 | { sizeof(cl_mem), &deshake_ctx->refined_features }, |
| | 1830 | { sizeof(cl_int), &deshake_ctx->refine_features } |
| | 1831 | ); |
| | 1832 | |
| | 1833 | CL_RUN_KERNEL_WITH_ARGS( |
| | 1834 | deshake_ctx->command_queue, |
| | 1835 | deshake_ctx->kernel_brief_descriptors, |
| | 1836 | grid_32_global_work, |
| | 1837 | NULL, |
| | 1838 | &brief_event, |
| | 1839 | { sizeof(cl_mem), &deshake_ctx->grayscale }, |
| | 1840 | { sizeof(cl_mem), &deshake_ctx->refined_features }, |
| | 1841 | { sizeof(cl_mem), &deshake_ctx->descriptors }, |
| | 1842 | { sizeof(cl_mem), &deshake_ctx->brief_pattern} |
| | 1843 | ); |
| | 1844 | |
| | 1845 | if (av_fifo_size(deshake_ctx->abs_motion.ringbuffers[RingbufX]) == 0) { |
| | 1846 | // This is the first frame we've been given to queue, meaning there is |
| | 1847 | // no previous frame to match descriptors to |
| | 1848 | |
| | 1849 | goto no_motion_data; |
| | 1850 | } |
| | 1851 | |
| | 1852 | CL_RUN_KERNEL_WITH_ARGS( |
| | 1853 | deshake_ctx->command_queue, |
| | 1854 | deshake_ctx->kernel_match_descriptors, |
| | 1855 | grid_32_global_work, |
| | 1856 | NULL, |
| | 1857 | &match_descriptors_event, |
| | 1858 | { sizeof(cl_mem), &deshake_ctx->prev_refined_features }, |
| | 1859 | { sizeof(cl_mem), &deshake_ctx->refined_features }, |
| | 1860 | { sizeof(cl_mem), &deshake_ctx->descriptors }, |
| | 1861 | { sizeof(cl_mem), &deshake_ctx->prev_descriptors }, |
| | 1862 | { sizeof(cl_mem), &deshake_ctx->matches } |
| | 1863 | ); |
| | 1864 | |
| | 1865 | cle = clEnqueueReadBuffer( |
| | 1866 | deshake_ctx->command_queue, |
| | 1867 | deshake_ctx->matches, |
| | 1868 | CL_TRUE, |
| | 1869 | 0, |
| | 1870 | grid_32_h * grid_32_w * sizeof(MotionVector), |
| | 1871 | deshake_ctx->matches_host, |
| | 1872 | 0, |
| | 1873 | NULL, |
| | 1874 | &read_buf_event |
| | 1875 | ); |
| | 1876 | CL_FAIL_ON_ERROR(AVERROR(EIO), "Failed to read matches to host: %d.\n", cle); |
| | 1877 | |
| | 1878 | num_vectors = make_vectors_contig(deshake_ctx, grid_32_h, grid_32_w); |
| | 1879 | |
| | 1880 | if (num_vectors < 10) { |
| | 1881 | // Not enough matches to get reliable motion data for this frame |
| | 1882 | // |
| | 1883 | // From this point on all data is relative to this frame rather than the |
| | 1884 | // original frame. We have to make sure that we don't mix values that were |
| | 1885 | // relative to the original frame with the new values relative to this |
| | 1886 | // frame when doing the gaussian smoothing. We keep track of where the old |
| | 1887 | // values end using this data_end_offset field in order to accomplish |
| | 1888 | // that goal. |
| | 1889 | // |
| | 1890 | // If no motion data is present for multiple frames in a short window of |
| | 1891 | // time, we leave the end where it was to avoid mixing 0s in with the |
| | 1892 | // old data (and just treat them all as part of the new values) |
| | 1893 | if (deshake_ctx->abs_motion.data_end_offset == -1) { |
| | 1894 | deshake_ctx->abs_motion.data_end_offset = |
| | 1895 | av_fifo_size(deshake_ctx->abs_motion.ringbuffers[RingbufX]) / sizeof(float) - 1; |
| | 1896 | } |
| | 1897 | |
| | 1898 | goto no_motion_data; |
| | 1899 | } |
| | 1900 | |
| | 1901 | if (!estimate_affine_2d( |
| | 1902 | deshake_ctx, |
| | 1903 | deshake_ctx->matches_contig_host, |
| | 1904 | &debug_matches, |
| | 1905 | num_vectors, |
| | 1906 | model.matrix, |
| | 1907 | 10.0, |
| | 1908 | 3000, |
| | 1909 | 0.999999999999 |
| | 1910 | )) { |
| | 1911 | goto no_motion_data; |
| | 1912 | } |
| | 1913 | |
| | 1914 | for (int i = 0; i < num_vectors; i++) { |
| | 1915 | if (deshake_ctx->matches_contig_host[i].should_consider) { |
| | 1916 | deshake_ctx->inliers[num_inliers] = deshake_ctx->matches_contig_host[i]; |
| | 1917 | num_inliers++; |
| | 1918 | } |
| | 1919 | } |
| | 1920 | |
| | 1921 | if (!minimize_error( |
| | 1922 | deshake_ctx, |
| | 1923 | deshake_ctx->inliers, |
| | 1924 | &debug_matches, |
| | 1925 | num_inliers, |
| | 1926 | model.matrix, |
| | 1927 | 400 |
| | 1928 | )) { |
| | 1929 | goto no_motion_data; |
| | 1930 | } |
| | 1931 | |
| | 1932 | |
| | 1933 | relative = decompose_transform(model.matrix); |
| | 1934 | |
| | 1935 | // Get the absolute transform data for the previous frame |
| | 1936 | for (int i = 0; i < RingbufCount; i++) { |
| | 1937 | av_fifo_generic_peek_at( |
| | 1938 | deshake_ctx->abs_motion.ringbuffers[i], |
| | 1939 | &prev_vals[i], |
| | 1940 | av_fifo_size(deshake_ctx->abs_motion.ringbuffers[i]) - sizeof(float), |
| | 1941 | sizeof(float), |
| | 1942 | NULL |
| | 1943 | ); |
| | 1944 | } |
| | 1945 | |
| | 1946 | new_vals[RingbufX] = prev_vals[RingbufX] + relative.translation.s[0]; |
| | 1947 | new_vals[RingbufY] = prev_vals[RingbufY] + relative.translation.s[1]; |
| | 1948 | new_vals[RingbufRot] = prev_vals[RingbufRot] + relative.rotation; |
| | 1949 | new_vals[RingbufScaleX] = prev_vals[RingbufScaleX] / relative.scale.s[0]; |
| | 1950 | new_vals[RingbufScaleY] = prev_vals[RingbufScaleY] / relative.scale.s[1]; |
| | 1951 | |
| | 1952 | if (deshake_ctx->debug_on) { |
| | 1953 | if (!deshake_ctx->is_yuv) { |
| | 1954 | deshake_ctx->grayscale_time += ff_opencl_get_event_time(grayscale_event); |
| | 1955 | } |
| | 1956 | deshake_ctx->harris_response_time += ff_opencl_get_event_time(harris_response_event); |
| | 1957 | deshake_ctx->refine_features_time += ff_opencl_get_event_time(refine_features_event); |
| | 1958 | deshake_ctx->brief_descriptors_time += ff_opencl_get_event_time(brief_event); |
| | 1959 | deshake_ctx->match_descriptors_time += ff_opencl_get_event_time(match_descriptors_event); |
| | 1960 | deshake_ctx->read_buf_time += ff_opencl_get_event_time(read_buf_event); |
| | 1961 | } |
| | 1962 | |
| | 1963 | goto end; |
| | 1964 | |
| | 1965 | no_motion_data: |
| | 1966 | new_vals[RingbufX] = 0.0f; |
| | 1967 | new_vals[RingbufY] = 0.0f; |
| | 1968 | new_vals[RingbufRot] = 0.0f; |
| | 1969 | new_vals[RingbufScaleX] = 1.0f; |
| | 1970 | new_vals[RingbufScaleY] = 1.0f; |
| | 1971 | |
| | 1972 | for (int i = 0; i < num_vectors; i++) { |
| | 1973 | deshake_ctx->matches_contig_host[i].should_consider = false; |
| | 1974 | } |
| | 1975 | debug_matches.num_model_matches = 0; |
| | 1976 | |
| | 1977 | if (deshake_ctx->debug_on) { |
| | 1978 | av_log(avctx, AV_LOG_VERBOSE, |
| | 1979 | "\n[ALERT] No motion data found in queue_frame, motion reset to 0\n\n" |
| | 1980 | ); |
| | 1981 | } |
| | 1982 | |
| | 1983 | goto end; |
| | 1984 | |
| | 1985 | end: |
| | 1986 | // Swap the descriptor buffers (we don't need the previous frame's descriptors |
| | 1987 | // again so we will use that space for the next frame's descriptors) |
| | 1988 | temp = deshake_ctx->prev_descriptors; |
| | 1989 | deshake_ctx->prev_descriptors = deshake_ctx->descriptors; |
| | 1990 | deshake_ctx->descriptors = temp; |
| | 1991 | |
| | 1992 | // Same for the refined features |
| | 1993 | temp = deshake_ctx->prev_refined_features; |
| | 1994 | deshake_ctx->prev_refined_features = deshake_ctx->refined_features; |
| | 1995 | deshake_ctx->refined_features = temp; |
| | 1996 | |
| | 1997 | if (deshake_ctx->debug_on) { |
| | 1998 | if (num_vectors == 0) { |
| | 1999 | debug_matches.matches = NULL; |
| | 2000 | } else { |
| | 2001 | debug_matches.matches = av_malloc_array(num_vectors, sizeof(MotionVector)); |
| | 2002 | |
| | 2003 | if (!debug_matches.matches) { |
| | 2004 | err = AVERROR(ENOMEM); |
| | 2005 | goto fail; |
| | 2006 | } |
| | 2007 | } |
| | 2008 | |
| | 2009 | for (int i = 0; i < num_vectors; i++) { |
| | 2010 | debug_matches.matches[i] = deshake_ctx->matches_contig_host[i]; |
| | 2011 | } |
| | 2012 | debug_matches.num_matches = num_vectors; |
| | 2013 | |
| | 2014 | av_fifo_generic_write( |
| | 2015 | deshake_ctx->abs_motion.debug_matches, |
| | 2016 | &debug_matches, |
| | 2017 | sizeof(DebugMatches), |
| | 2018 | NULL |
| | 2019 | ); |
| | 2020 | } |
| | 2021 | |
| | 2022 | for (int i = 0; i < RingbufCount; i++) { |
| | 2023 | av_fifo_generic_write( |
| | 2024 | deshake_ctx->abs_motion.ringbuffers[i], |
| | 2025 | &new_vals[i], |
| | 2026 | sizeof(float), |
| | 2027 | NULL |
| | 2028 | ); |
| | 2029 | } |
| | 2030 | |
| | 2031 | return ff_framequeue_add(&deshake_ctx->fq, input_frame); |
| | 2032 | |
| | 2033 | fail: |
| | 2034 | clFinish(deshake_ctx->command_queue); |
| | 2035 | av_frame_free(&input_frame); |
| | 2036 | return err; |
| | 2037 | } |
| | 2038 | |
| | 2039 | static int activate(AVFilterContext *ctx) |
| | 2040 | { |
| | 2041 | AVFilterLink *inlink = ctx->inputs[0]; |
| | 2042 | AVFilterLink *outlink = ctx->outputs[0]; |
| | 2043 | DeshakeOpenCLContext *deshake_ctx = ctx->priv; |
| | 2044 | AVFrame *frame = NULL; |
| | 2045 | int ret, status; |
| | 2046 | int64_t pts; |
| | 2047 | |
| | 2048 | FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink); |
| | 2049 | |
| | 2050 | if (!deshake_ctx->eof) { |
| | 2051 | ret = ff_inlink_consume_frame(inlink, &frame); |
| | 2052 | if (ret < 0) |
| | 2053 | return ret; |
| | 2054 | if (ret > 0) { |
| | 2055 | if (!frame->hw_frames_ctx) |
| | 2056 | return AVERROR(EINVAL); |
| | 2057 | |
| | 2058 | if (!deshake_ctx->initialized) { |
| | 2059 | ret = deshake_opencl_init(ctx); |
| | 2060 | if (ret < 0) |
| | 2061 | return ret; |
| | 2062 | } |
| | 2063 | |
| | 2064 | // If there is no more space in the ringbuffers, remove the oldest |
| | 2065 | // values to make room for the new ones |
| | 2066 | if (av_fifo_space(deshake_ctx->abs_motion.ringbuffers[RingbufX]) == 0) { |
| | 2067 | for (int i = 0; i < RingbufCount; i++) { |
| | 2068 | av_fifo_drain(deshake_ctx->abs_motion.ringbuffers[i], sizeof(float)); |
| | 2069 | } |
| | 2070 | } |
| | 2071 | ret = queue_frame(inlink, frame); |
| | 2072 | if (ret < 0) |
| | 2073 | return ret; |
| | 2074 | if (ret >= 0) { |
| | 2075 | // See if we have enough buffered frames to process one |
| | 2076 | // |
| | 2077 | // "enough" is half the smooth window of queued frames into the future |
| | 2078 | if (ff_framequeue_queued_frames(&deshake_ctx->fq) >= deshake_ctx->smooth_window / 2) { |
| | 2079 | return filter_frame(inlink, ff_framequeue_take(&deshake_ctx->fq)); |
| | 2080 | } |
| | 2081 | } |
| | 2082 | } |
| | 2083 | } |
| | 2084 | |
| | 2085 | if (!deshake_ctx->eof && ff_inlink_acknowledge_status(inlink, &status, &pts)) { |
| | 2086 | if (status == AVERROR_EOF) { |
| | 2087 | deshake_ctx->eof = true; |
| | 2088 | } |
| | 2089 | } |
| | 2090 | |
| | 2091 | if (deshake_ctx->eof) { |
| | 2092 | // Finish processing the rest of the frames in the queue. |
| | 2093 | while(ff_framequeue_queued_frames(&deshake_ctx->fq) != 0) { |
| | 2094 | for (int i = 0; i < RingbufCount; i++) { |
| | 2095 | av_fifo_drain(deshake_ctx->abs_motion.ringbuffers[i], sizeof(float)); |
| | 2096 | } |
| | 2097 | |
| | 2098 | ret = filter_frame(inlink, ff_framequeue_take(&deshake_ctx->fq)); |
| | 2099 | if (ret < 0) { |
| | 2100 | return ret; |
| | 2101 | } |
| | 2102 | } |
| | 2103 | |
| | 2104 | if (deshake_ctx->debug_on) { |
| | 2105 | av_log(ctx, AV_LOG_VERBOSE, |
| | 2106 | "Average kernel execution times:\n" |
| | 2107 | "\t grayscale: %0.3f ms\n" |
| | 2108 | "\t harris_response: %0.3f ms\n" |
| | 2109 | "\t refine_features: %0.3f ms\n" |
| | 2110 | "\tbrief_descriptors: %0.3f ms\n" |
| | 2111 | "\tmatch_descriptors: %0.3f ms\n" |
| | 2112 | "\t transform: %0.3f ms\n" |
| | 2113 | "\t crop_upscale: %0.3f ms\n" |
| | 2114 | "Average buffer read times:\n" |
| | 2115 | "\t features buf: %0.3f ms\n", |
| | 2116 | averaged_event_time_ms(deshake_ctx->grayscale_time, deshake_ctx->curr_frame), |
| | 2117 | averaged_event_time_ms(deshake_ctx->harris_response_time, deshake_ctx->curr_frame), |
| | 2118 | averaged_event_time_ms(deshake_ctx->refine_features_time, deshake_ctx->curr_frame), |
| | 2119 | averaged_event_time_ms(deshake_ctx->brief_descriptors_time, deshake_ctx->curr_frame), |
| | 2120 | averaged_event_time_ms(deshake_ctx->match_descriptors_time, deshake_ctx->curr_frame), |
| | 2121 | averaged_event_time_ms(deshake_ctx->transform_time, deshake_ctx->curr_frame), |
| | 2122 | averaged_event_time_ms(deshake_ctx->crop_upscale_time, deshake_ctx->curr_frame), |
| | 2123 | averaged_event_time_ms(deshake_ctx->read_buf_time, deshake_ctx->curr_frame) |
| | 2124 | ); |
| | 2125 | } |
| | 2126 | |
| | 2127 | ff_outlink_set_status(outlink, AVERROR_EOF, deshake_ctx->duration); |
| | 2128 | return 0; |
| | 2129 | } |
| | 2130 | |
| | 2131 | if (!deshake_ctx->eof) { |
| | 2132 | FF_FILTER_FORWARD_WANTED(outlink, inlink); |
| | 2133 | } |
| | 2134 | |
| | 2135 | return FFERROR_NOT_READY; |
| | 2136 | } |
| | 2137 | |
| | 2138 | static const AVFilterPad deshake_opencl_inputs[] = { |
| | 2139 | { |
| | 2140 | .name = "default", |
| | 2141 | .type = AVMEDIA_TYPE_VIDEO, |
| | 2142 | .config_props = &ff_opencl_filter_config_input, |
| | 2143 | }, |
| | 2144 | { NULL } |
| | 2145 | }; |
| | 2146 | |
| | 2147 | static const AVFilterPad deshake_opencl_outputs[] = { |
| | 2148 | { |
| | 2149 | .name = "default", |
| | 2150 | .type = AVMEDIA_TYPE_VIDEO, |
| | 2151 | .config_props = &ff_opencl_filter_config_output, |
| | 2152 | }, |
| | 2153 | { NULL } |
| | 2154 | }; |
| | 2155 | |
| | 2156 | #define OFFSET(x) offsetof(DeshakeOpenCLContext, x) |
| | 2157 | #define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM |
| | 2158 | |
| | 2159 | static const AVOption deshake_opencl_options[] = { |
| | 2160 | { |
| | 2161 | "tripod", "simulates a tripod by preventing any camera movement whatsoever " |
| | 2162 | "from the original frame", |
| | 2163 | OFFSET(tripod_mode), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, FLAGS |
| | 2164 | }, |
| | 2165 | { |
| | 2166 | "debug", "turn on additional debugging information", |
| | 2167 | OFFSET(debug_on), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, FLAGS |
| | 2168 | }, |
| | 2169 | { |
| | 2170 | "adaptive_crop", "attempt to subtly crop borders to reduce mirrored content", |
| | 2171 | OFFSET(should_crop), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, FLAGS |
| | 2172 | }, |
| | 2173 | { |
| | 2174 | "refine_features", "refine feature point locations at a sub-pixel level", |
| | 2175 | OFFSET(refine_features), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, FLAGS |
| | 2176 | }, |
| | 2177 | { |
| | 2178 | "smooth_strength", "smoothing strength (0 attempts to adaptively determine optimal strength)", |
| | 2179 | OFFSET(smooth_percent), AV_OPT_TYPE_FLOAT, {.dbl = 0.0f}, 0.0f, 1.0f, FLAGS |
| | 2180 | }, |
| | 2181 | { |
| | 2182 | "smooth_window_multiplier", "multiplier for number of frames to buffer for motion data", |
| | 2183 | OFFSET(smooth_window_multiplier), AV_OPT_TYPE_FLOAT, {.dbl = 2.0}, 0.1, 10.0, FLAGS |
| | 2184 | }, |
| | 2185 | { NULL } |
| | 2186 | }; |
| | 2187 | |
| | 2188 | AVFILTER_DEFINE_CLASS(deshake_opencl); |
| | 2189 | |
| | 2190 | AVFilter ff_vf_deshake_opencl = { |
| | 2191 | .name = "deshake_opencl", |
| | 2192 | .description = NULL_IF_CONFIG_SMALL("Feature-point based video stabilization filter"), |
| | 2193 | .priv_size = sizeof(DeshakeOpenCLContext), |
| | 2194 | .priv_class = &deshake_opencl_class, |
| | 2195 | .init = &ff_opencl_filter_init, |
| | 2196 | .uninit = &deshake_opencl_uninit, |
| | 2197 | .query_formats = &ff_opencl_filter_query_formats, |
| | 2198 | .activate = activate, |
| | 2199 | .inputs = deshake_opencl_inputs, |
| | 2200 | .outputs = deshake_opencl_outputs, |
| | 2201 | .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE |
| | 2202 | }; |