History

lavfi: add deshake_opencl filter

Jarek Samic committed 7 years ago

Commit b29c7bc1 parent 5b5746b8 files · +2924 −1Browse files

Showing the changes to libavfilter (6 of 8 files). Show all

Unified · Split

6 files
  1. libavfilter/Makefile +2 −0
  2. libavfilter/allfilters.c +1 −0
  3. libavfilter/opencl/deshake.cl +647 −0
  4. libavfilter/opencl_source.h +1 −0
  5. libavfilter/version.h +1 −1
  6. libavfilter/vf_deshake_opencl.c +2202 −0
libavfilter/Makefile+2−0
@@ -211,6 +211,8 @@ OBJS-$(CONFIG_DEINTERLACE_VAAPI_FILTER) += vf_deinterlace_vaapi.o vaapi_vpp · 210 unmodified lines
211211 OBJS-$(CONFIG_DEJUDDER_FILTER) += vf_dejudder.o
212212 OBJS-$(CONFIG_DELOGO_FILTER) += vf_delogo.o
213213 OBJS-$(CONFIG_DENOISE_VAAPI_FILTER) += vf_misc_vaapi.o vaapi_vpp.o
214+OBJS-$(CONFIG_DESHAKE_OPENCL_FILTER) += vf_deshake_opencl.o opencl.o \
215+ opencl/deshake.o
214216 OBJS-$(CONFIG_DESHAKE_FILTER) += vf_deshake.o
215217 OBJS-$(CONFIG_DESPILL_FILTER) += vf_despill.o
216218 OBJS-$(CONFIG_DETELECINE_FILTER) += vf_detelecine.o
libavfilter/allfilters.c+1−0
@@ -200,6 +200,7 @@ extern AVFilter ff_vf_delogo; · 199 unmodified lines
200200 extern AVFilter ff_vf_denoise_vaapi;
201201 extern AVFilter ff_vf_derain;
202202 extern AVFilter ff_vf_deshake;
203+extern AVFilter ff_vf_deshake_opencl;
203204 extern AVFilter ff_vf_despill;
204205 extern AVFilter ff_vf_detelecine;
205206 extern AVFilter ff_vf_dilation;
libavfilter/opencl/deshake.cl+647−0
@@ -0,0 +1,647 @@
1+/*
2+ * This file is part of FFmpeg.
3+ *
4+ * FFmpeg is free software; you can redistribute it and/or
5+ * modify it under the terms of the GNU Lesser General Public
6+ * License as published by the Free Software Foundation; either
7+ * version 2.1 of the License, or (at your option) any later version.
8+ *
9+ * FFmpeg is distributed in the hope that it will be useful,
10+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
11+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12+ * Lesser General Public License for more details.
13+ *
14+ * You should have received a copy of the GNU Lesser General Public
15+ * License along with FFmpeg; if not, write to the Free Software
16+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17+ *
18+ * Copyright (C) 2000, Intel Corporation, all rights reserved.
19+ * Copyright (C) 2013, OpenCV Foundation, all rights reserved.
20+ * Third party copyrights are property of their respective owners.
21+ *
22+ * Redistribution and use in source and binary forms, with or without modification,
23+ * are permitted provided that the following conditions are met:
24+ *
25+ * * Redistribution's of source code must retain the above copyright notice,
26+ * this list of conditions and the following disclaimer.
27+ *
28+ * * Redistribution's in binary form must reproduce the above copyright notice,
29+ * this list of conditions and the following disclaimer in the documentation
30+ * and/or other materials provided with the distribution.
31+ *
32+ * * The name of the copyright holders may not be used to endorse or promote products
33+ * derived from this software without specific prior written permission.
34+ *
35+ * This software is provided by the copyright holders and contributors "as is" and
36+ * any express or implied warranties, including, but not limited to, the implied
37+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
38+ * In no event shall the Intel Corporation or contributors be liable for any direct,
39+ * indirect, incidental, special, exemplary, or consequential damages
40+ * (including, but not limited to, procurement of substitute goods or services;
41+ * loss of use, data, or profits; or business interruption) however caused
42+ * and on any theory of liability, whether in contract, strict liability,
43+ * or tort (including negligence or otherwise) arising in any way out of
44+ * the use of this software, even if advised of the possibility of such damage.
45+ */
46+
47+#define HARRIS_THRESHOLD 3.0f
48+// Block size over which to compute harris response
49+//
50+// Note that changing this will require fiddling with the local array sizes in
51+// harris_response
52+#define HARRIS_RADIUS 2
53+#define DISTANCE_THRESHOLD 80
54+
55+// Sub-pixel refinement window for feature points
56+#define REFINE_WIN_HALF_W 5
57+#define REFINE_WIN_HALF_H 5
58+#define REFINE_WIN_W 11 // REFINE_WIN_HALF_W * 2 + 1
59+#define REFINE_WIN_H 11
60+
61+// Non-maximum suppression window size
62+#define NONMAX_WIN 30
63+#define NONMAX_WIN_HALF 15 // NONMAX_WIN / 2
64+
65+typedef struct PointPair {
66+ // Previous frame
67+ float2 p1;
68+ // Current frame
69+ float2 p2;
70+} PointPair;
71+
72+typedef struct SmoothedPointPair {
73+ // Non-smoothed point in current frame
74+ int2 p1;
75+ // Smoothed point in current frame
76+ float2 p2;
77+} SmoothedPointPair;
78+
79+typedef struct MotionVector {
80+ PointPair p;
81+ // Used to mark vectors as potential outliers
82+ int should_consider;
83+} MotionVector;
84+
85+const sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE |
86+ CLK_ADDRESS_CLAMP_TO_EDGE |
87+ CLK_FILTER_NEAREST;
88+
89+const sampler_t sampler_linear = CLK_NORMALIZED_COORDS_FALSE |
90+ CLK_ADDRESS_CLAMP_TO_EDGE |
91+ CLK_FILTER_LINEAR;
92+
93+const sampler_t sampler_linear_mirror = CLK_NORMALIZED_COORDS_TRUE |
94+ CLK_ADDRESS_MIRRORED_REPEAT |
95+ CLK_FILTER_LINEAR;
96+
97+// Writes to a 1D array at loc, treating it as a 2D array with the same
98+// dimensions as the global work size.
99+static void write_to_1d_arrf(__global float *buf, int2 loc, float val) {
100+ buf[loc.x + loc.y * get_global_size(0)] = val;
101+}
102+
103+static void write_to_1d_arrul8(__global ulong8 *buf, int2 loc, ulong8 val) {
104+ buf[loc.x + loc.y * get_global_size(0)] = val;
105+}
106+
107+static void write_to_1d_arrvec(__global MotionVector *buf, int2 loc, MotionVector val) {
108+ buf[loc.x + loc.y * get_global_size(0)] = val;
109+}
110+
111+static void write_to_1d_arrf2(__global float2 *buf, int2 loc, float2 val) {
112+ buf[loc.x + loc.y * get_global_size(0)] = val;
113+}
114+
115+static ulong8 read_from_1d_arrul8(__global const ulong8 *buf, int2 loc) {
116+ return buf[loc.x + loc.y * get_global_size(0)];
117+}
118+
119+static float2 read_from_1d_arrf2(__global const float2 *buf, int2 loc) {
120+ return buf[loc.x + loc.y * get_global_size(0)];
121+}
122+
123+// Returns the grayscale value at the given point.
124+static float pixel_grayscale(__read_only image2d_t src, int2 loc) {
125+ float4 pixel = read_imagef(src, sampler, loc);
126+ return (pixel.x + pixel.y + pixel.z) / 3.0f;
127+}
128+
129+static float convolve(
130+ __local const float *grayscale,
131+ int local_idx_x,
132+ int local_idx_y,
133+ float mask[3][3]
134+) {
135+ float ret = 0;
136+
137+ // These loops touch each pixel surrounding loc as well as loc itself
138+ for (int i = 1, i2 = 0; i >= -1; --i, ++i2) {
139+ for (int j = -1, j2 = 0; j <= 1; ++j, ++j2) {
140+ ret += mask[i2][j2] * grayscale[(local_idx_x + 3 + j) + (local_idx_y + 3 + i) * 14];
141+ }
142+ }
143+
144+ return ret;
145+}
146+
147+// Sums dx * dy for all pixels within radius of loc
148+static float sum_deriv_prod(
149+ __local const float *grayscale,
150+ float mask_x[3][3],
151+ float mask_y[3][3]
152+) {
153+ float ret = 0;
154+
155+ for (int i = HARRIS_RADIUS; i >= -HARRIS_RADIUS; --i) {
156+ for (int j = -HARRIS_RADIUS; j <= HARRIS_RADIUS; ++j) {
157+ ret += convolve(grayscale, get_local_id(0) + j, get_local_id(1) + i, mask_x) *
158+ convolve(grayscale, get_local_id(0) + j, get_local_id(1) + i, mask_y);
159+ }
160+ }
161+
162+ return ret;
163+}
164+
165+// Sums d<>^2 (determined by mask) for all pixels within radius of loc
166+static float sum_deriv_pow(__local const float *grayscale, float mask[3][3])
167+{
168+ float ret = 0;
169+
170+ for (int i = HARRIS_RADIUS; i >= -HARRIS_RADIUS; --i) {
171+ for (int j = -HARRIS_RADIUS; j <= HARRIS_RADIUS; ++j) {
172+ float deriv = convolve(grayscale, get_local_id(0) + j, get_local_id(1) + i, mask);
173+ ret += deriv * deriv;
174+ }
175+ }
176+
177+ return ret;
178+}
179+
180+// Fills a box with the given radius and pixel around loc
181+static void draw_box(__write_only image2d_t dst, int2 loc, float4 pixel, int radius)
182+{
183+ for (int i = -radius; i <= radius; ++i) {
184+ for (int j = -radius; j <= radius; ++j) {
185+ write_imagef(
186+ dst,
187+ (int2)(
188+ // Clamp to avoid writing outside image bounds
189+ clamp(loc.x + i, 0, get_image_dim(dst).x - 1),
190+ clamp(loc.y + j, 0, get_image_dim(dst).y - 1)
191+ ),
192+ pixel
193+ );
194+ }
195+ }
196+}
197+
198+// Converts the src image to grayscale
199+__kernel void grayscale(
200+ __read_only image2d_t src,
201+ __write_only image2d_t grayscale
202+) {
203+ int2 loc = (int2)(get_global_id(0), get_global_id(1));
204+ write_imagef(grayscale, loc, (float4)(pixel_grayscale(src, loc), 0.0f, 0.0f, 1.0f));
205+}
206+
207+// This kernel computes the harris response for the given grayscale src image
208+// within the given radius and writes it to harris_buf
209+__kernel void harris_response(
210+ __read_only image2d_t grayscale,
211+ __global float *harris_buf
212+) {
213+ int2 loc = (int2)(get_global_id(0), get_global_id(1));
214+
215+ if (loc.x > get_image_width(grayscale) - 1 || loc.y > get_image_height(grayscale) - 1) {
216+ write_to_1d_arrf(harris_buf, loc, 0);
217+ return;
218+ }
219+
220+ float scale = 1.0f / ((1 << 2) * HARRIS_RADIUS * 255.0f);
221+
222+ float sobel_mask_x[3][3] = {
223+ {-1, 0, 1},
224+ {-2, 0, 2},
225+ {-1, 0, 1}
226+ };
227+
228+ float sobel_mask_y[3][3] = {
229+ { 1, 2, 1},
230+ { 0, 0, 0},
231+ {-1, -2, -1}
232+ };
233+
234+ // 8 x 8 local work + 3 pixels around each side (needed to accomodate for the
235+ // block size radius of 2)
236+ __local float grayscale_data[196];
237+
238+ int idx = get_group_id(0) * get_local_size(0);
239+ int idy = get_group_id(1) * get_local_size(1);
240+
241+ for (int i = idy - 3, it = 0; i < idy + (int)get_local_size(1) + 3; i++, it++) {
242+ for (int j = idx - 3, jt = 0; j < idx + (int)get_local_size(0) + 3; j++, jt++) {
243+ grayscale_data[jt + it * 14] = read_imagef(grayscale, sampler, (int2)(j, i)).x;
244+ }
245+ }
246+
247+ barrier(CLK_LOCAL_MEM_FENCE);
248+
249+ float sumdxdy = sum_deriv_prod(grayscale_data, sobel_mask_x, sobel_mask_y);
250+ float sumdx2 = sum_deriv_pow(grayscale_data, sobel_mask_x);
251+ float sumdy2 = sum_deriv_pow(grayscale_data, sobel_mask_y);
252+
253+ float trace = sumdx2 + sumdy2;
254+ // r = det(M) - k(trace(M))^2
255+ // k usually between 0.04 to 0.06
256+ float r = (sumdx2 * sumdy2 - sumdxdy * sumdxdy) - 0.04f * (trace * trace) * pown(scale, 4);
257+
258+ // Threshold the r value
259+ harris_buf[loc.x + loc.y * get_image_width(grayscale)] = r * step(HARRIS_THRESHOLD, r);
260+}
261+
262+// Gets a patch centered around a float coordinate from a grayscale image using
263+// bilinear interpolation
264+static void get_rect_sub_pix(
265+ __read_only image2d_t grayscale,
266+ float *buffer,
267+ int size_x,
268+ int size_y,
269+ float2 center
270+) {
271+ float2 offset = ((float2)(size_x, size_y) - 1.0f) * 0.5f;
272+
273+ for (int i = 0; i < size_y; i++) {
274+ for (int j = 0; j < size_x; j++) {
275+ buffer[i * size_x + j] = read_imagef(
276+ grayscale,
277+ sampler_linear,
278+ (float2)(j, i) + center - offset
279+ ).x * 255.0f;
280+ }
281+ }
282+}
283+
284+// Refines detected features at a sub-pixel level
285+//
286+// This function is ported from OpenCV
287+static float2 corner_sub_pix(
288+ __read_only image2d_t grayscale,
289+ float2 feature,
290+ float *mask
291+) {
292+ float2 init = feature;
293+ int src_width = get_global_size(0);
294+ int src_height = get_global_size(1);
295+
296+ const int max_iters = 40;
297+ const float eps = 0.001f * 0.001f;
298+ int i, j, k;
299+
300+ int iter = 0;
301+ float err = 0;
302+ float subpix[(REFINE_WIN_W + 2) * (REFINE_WIN_H + 2)];
303+ const float flt_epsilon = 0x1.0p-23f;
304+
305+ do {
306+ float2 feature_tmp;
307+ float a = 0, b = 0, c = 0, bb1 = 0, bb2 = 0;
308+
309+ get_rect_sub_pix(grayscale, subpix, REFINE_WIN_W + 2, REFINE_WIN_H + 2, feature);
310+ float *subpix_ptr = subpix;
311+ subpix_ptr += REFINE_WIN_W + 2 + 1;
312+
313+ // process gradient
314+ for (i = 0, k = 0; i < REFINE_WIN_H; i++, subpix_ptr += REFINE_WIN_W + 2) {
315+ float py = i - REFINE_WIN_HALF_H;
316+
317+ for (j = 0; j < REFINE_WIN_W; j++, k++) {
318+ float m = mask[k];
319+ float tgx = subpix_ptr[j + 1] - subpix_ptr[j - 1];
320+ float tgy = subpix_ptr[j + REFINE_WIN_W + 2] - subpix_ptr[j - REFINE_WIN_W - 2];
321+ float gxx = tgx * tgx * m;
322+ float gxy = tgx * tgy * m;
323+ float gyy = tgy * tgy * m;
324+ float px = j - REFINE_WIN_HALF_W;
325+
326+ a += gxx;
327+ b += gxy;
328+ c += gyy;
329+
330+ bb1 += gxx * px + gxy * py;
331+ bb2 += gxy * px + gyy * py;
332+ }
333+ }
334+
335+ float det = a * c - b * b;
336+ if (fabs(det) <= flt_epsilon * flt_epsilon) {
337+ break;
338+ }
339+
340+ // 2x2 matrix inversion
341+ float scale = 1.0f / det;
342+ feature_tmp.x = (float)(feature.x + (c * scale * bb1) - (b * scale * bb2));
343+ feature_tmp.y = (float)(feature.y - (b * scale * bb1) + (a * scale * bb2));
344+ err = dot(feature_tmp - feature, feature_tmp - feature);
345+
346+ feature = feature_tmp;
347+ if (feature.x < 0 || feature.x >= src_width || feature.y < 0 || feature.y >= src_height) {
348+ break;
349+ }
350+ } while (++iter < max_iters && err > eps);
351+
352+ // Make sure new point isn't too far from the initial point (indicates poor convergence)
353+ if (fabs(feature.x - init.x) > REFINE_WIN_HALF_W || fabs(feature.y - init.y) > REFINE_WIN_HALF_H) {
354+ feature = init;
355+ }
356+
357+ return feature;
358+}
359+
360+// Performs non-maximum suppression on the harris response and writes the resulting
361+// feature locations to refined_features.
362+//
363+// Assumes that refined_features and the global work sizes are set up such that the image
364+// is split up into a grid of 32x32 blocks where each block has a single slot in the
365+// refined_features buffer. This kernel finds the best corner in each block (if the
366+// block has any) and writes it to the corresponding slot in the buffer.
367+//
368+// If subpixel_refine is true, the features are additionally refined at a sub-pixel
369+// level for increased precision.
370+__kernel void refine_features(
371+ __read_only image2d_t grayscale,
372+ __global const float *harris_buf,
373+ __global float2 *refined_features,
374+ int subpixel_refine
375+) {
376+ int2 loc = (int2)(get_global_id(0), get_global_id(1));
377+ // The location in the grayscale buffer rather than the compacted grid
378+ int2 loc_i = (int2)(loc.x * 32, loc.y * 32);
379+
380+ float new_val;
381+ float max_val = 0;
382+ float2 loc_max = (float2)(-1, -1);
383+
384+ int end_x = min(loc_i.x + 32, (int)get_image_dim(grayscale).x - 1);
385+ int end_y = min(loc_i.y + 32, (int)get_image_dim(grayscale).y - 1);
386+
387+ for (int i = loc_i.x; i < end_x; ++i) {
388+ for (int j = loc_i.y; j < end_y; ++j) {
389+ new_val = harris_buf[i + j * get_image_dim(grayscale).x];
390+
391+ if (new_val > max_val) {
392+ max_val = new_val;
393+ loc_max = (float2)(i, j);
394+ }
395+ }
396+ }
397+
398+ if (max_val == 0) {
399+ // There are no features in this part of the frame
400+ write_to_1d_arrf2(refined_features, loc, loc_max);
401+ return;
402+ }
403+
404+ if (subpixel_refine) {
405+ float mask[REFINE_WIN_H * REFINE_WIN_W];
406+ for (int i = 0; i < REFINE_WIN_H; i++) {
407+ float y = (float)(i - REFINE_WIN_HALF_H) / REFINE_WIN_HALF_H;
408+ float vy = exp(-y * y);
409+
410+ for (int j = 0; j < REFINE_WIN_W; j++) {
411+ float x = (float)(j - REFINE_WIN_HALF_W) / REFINE_WIN_HALF_W;
412+ mask[i * REFINE_WIN_W + j] = (float)(vy * exp(-x * x));
413+ }
414+ }
415+
416+ loc_max = corner_sub_pix(grayscale, loc_max, mask);
417+ }
418+
419+ write_to_1d_arrf2(refined_features, loc, loc_max);
420+}
421+
422+// Extracts BRIEF descriptors from the grayscale src image for the given features
423+// using the provided sampler.
424+__kernel void brief_descriptors(
425+ __read_only image2d_t grayscale,
426+ __global const float2 *refined_features,
427+ // for 512 bit descriptors
428+ __global ulong8 *desc_buf,
429+ __global const PointPair *brief_pattern
430+) {
431+ int2 loc = (int2)(get_global_id(0), get_global_id(1));
432+ float2 feature = read_from_1d_arrf2(refined_features, loc);
433+
434+ // There was no feature in this part of the frame
435+ if (feature.x == -1) {
436+ write_to_1d_arrul8(desc_buf, loc, (ulong8)(0));
437+ return;
438+ }
439+
440+ ulong8 desc = 0;
441+ ulong *p = &desc;
442+
443+ for (int i = 0; i < 8; ++i) {
444+ for (int j = 0; j < 64; ++j) {
445+ PointPair pair = brief_pattern[j * (i + 1)];
446+ float l1 = read_imagef(grayscale, sampler_linear, feature + pair.p1).x;
447+ float l2 = read_imagef(grayscale, sampler_linear, feature + pair.p2).x;
448+
449+ if (l1 < l2) {
450+ p[i] |= 1UL << j;
451+ }
452+ }
453+ }
454+
455+ write_to_1d_arrul8(desc_buf, loc, desc);
456+}
457+
458+// Given buffers with descriptors for the current and previous frame, determines
459+// which ones match, writing correspondences to matches_buf.
460+//
461+// Feature and descriptor buffers are assumed to be compacted (each element sourced
462+// from a 32x32 block in the frame being processed).
463+__kernel void match_descriptors(
464+ __global const float2 *prev_refined_features,
465+ __global const float2 *refined_features,
466+ __global const ulong8 *desc_buf,
467+ __global const ulong8 *prev_desc_buf,
468+ __global MotionVector *matches_buf
469+) {
470+ int2 loc = (int2)(get_global_id(0), get_global_id(1));
471+ ulong8 desc = read_from_1d_arrul8(desc_buf, loc);
472+ const int search_radius = 3;
473+
474+ MotionVector invalid_vector = (MotionVector) {
475+ (PointPair) {
476+ (float2)(-1, -1),
477+ (float2)(-1, -1)
478+ },
479+ 0
480+ };
481+
482+ if (desc.s0 == 0 && desc.s1 == 0) {
483+ // There was no feature in this part of the frame
484+ write_to_1d_arrvec(
485+ matches_buf,
486+ loc,
487+ invalid_vector
488+ );
489+ return;
490+ }
491+
492+ int2 start = max(loc - search_radius, 0);
493+ int2 end = min(loc + search_radius, (int2)(get_global_size(0) - 1, get_global_size(1) - 1));
494+
495+ for (int i = start.x; i < end.x; ++i) {
496+ for (int j = start.y; j < end.y; ++j) {
497+ int2 prev_point = (int2)(i, j);
498+ int total_dist = 0;
499+
500+ ulong8 prev_desc = read_from_1d_arrul8(prev_desc_buf, prev_point);
501+
502+ if (prev_desc.s0 == 0 && prev_desc.s1 == 0) {
503+ continue;
504+ }
505+
506+ ulong *prev_desc_p = &prev_desc;
507+ ulong *desc_p = &desc;
508+
509+ for (int i = 0; i < 8; i++) {
510+ total_dist += popcount(desc_p[i] ^ prev_desc_p[i]);
511+ }
512+
513+ if (total_dist < DISTANCE_THRESHOLD) {
514+ write_to_1d_arrvec(
515+ matches_buf,
516+ loc,
517+ (MotionVector) {
518+ (PointPair) {
519+ read_from_1d_arrf2(prev_refined_features, prev_point),
520+ read_from_1d_arrf2(refined_features, loc)
521+ },
522+ 1
523+ }
524+ );
525+
526+ return;
527+ }
528+ }
529+ }
530+
531+ // There is no found match for this point
532+ write_to_1d_arrvec(
533+ matches_buf,
534+ loc,
535+ invalid_vector
536+ );
537+}
538+
539+// Returns the position of the given point after the transform is applied
540+static float2 transformed_point(float2 p, __global const float *transform) {
541+ float2 ret;
542+
543+ ret.x = p.x * transform[0] + p.y * transform[1] + transform[2];
544+ ret.y = p.x * transform[3] + p.y * transform[4] + transform[5];
545+
546+ return ret;
547+}
548+
549+
550+// Performs the given transform on the src image
551+__kernel void transform(
552+ __read_only image2d_t src,
553+ __write_only image2d_t dst,
554+ __global const float *transform
555+) {
556+ int2 loc = (int2)(get_global_id(0), get_global_id(1));
557+ float2 norm = convert_float2(get_image_dim(src));
558+
559+ write_imagef(
560+ dst,
561+ loc,
562+ read_imagef(
563+ src,
564+ sampler_linear_mirror,
565+ transformed_point((float2)(loc.x, loc.y), transform) / norm
566+ )
567+ );
568+}
569+
570+// Returns the new location of the given point using the given crop bounding box
571+// and the width and height of the original frame.
572+static float2 cropped_point(
573+ float2 p,
574+ float2 top_left,
575+ float2 bottom_right,
576+ int2 orig_dim
577+) {
578+ float2 ret;
579+
580+ float crop_width = bottom_right.x - top_left.x;
581+ float crop_height = bottom_right.y - top_left.y;
582+
583+ float width_norm = p.x / (float)orig_dim.x;
584+ float height_norm = p.y / (float)orig_dim.y;
585+
586+ ret.x = (width_norm * crop_width) + top_left.x;
587+ ret.y = (height_norm * crop_height) + ((float)orig_dim.y - bottom_right.y);
588+
589+ return ret;
590+}
591+
592+// Upscales the given cropped region to the size of the original frame
593+__kernel void crop_upscale(
594+ __read_only image2d_t src,
595+ __write_only image2d_t dst,
596+ float2 top_left,
597+ float2 bottom_right
598+) {
599+ int2 loc = (int2)(get_global_id(0), get_global_id(1));
600+
601+ write_imagef(
602+ dst,
603+ loc,
604+ read_imagef(
605+ src,
606+ sampler_linear,
607+ cropped_point((float2)(loc.x, loc.y), top_left, bottom_right, get_image_dim(dst))
608+ )
609+ );
610+}
611+
612+// Draws boxes to represent the given point matches and uses the given transform
613+// and crop info to make sure their positions are accurate on the transformed frame.
614+//
615+// model_matches is an array of three points that were used by the RANSAC process
616+// to generate the given transform
617+__kernel void draw_debug_info(
618+ __write_only image2d_t dst,
619+ __global const MotionVector *matches,
620+ __global const MotionVector *model_matches,
621+ int num_model_matches,
622+ __global const float *transform
623+) {
624+ int loc = get_global_id(0);
625+ MotionVector vec = matches[loc];
626+ // Black box: matched point that RANSAC considered an outlier
627+ float4 big_rect_color = (float4)(0.1f, 0.1f, 0.1f, 1.0f);
628+
629+ if (vec.should_consider) {
630+ // Green box: matched point that RANSAC considered an inlier
631+ big_rect_color = (float4)(0.0f, 1.0f, 0.0f, 1.0f);
632+ }
633+
634+ for (int i = 0; i < num_model_matches; i++) {
635+ if (vec.p.p2.x == model_matches[i].p.p2.x && vec.p.p2.y == model_matches[i].p.p2.y) {
636+ // Orange box: point used to calculate model
637+ big_rect_color = (float4)(1.0f, 0.5f, 0.0f, 1.0f);
638+ }
639+ }
640+
641+ float2 transformed_p1 = transformed_point(vec.p.p1, transform);
642+ float2 transformed_p2 = transformed_point(vec.p.p2, transform);
643+
644+ draw_box(dst, (int2)(transformed_p2.x, transformed_p2.y), big_rect_color, 5);
645+ // Small light blue box: the point in the previous frame
646+ draw_box(dst, (int2)(transformed_p1.x, transformed_p1.y), (float4)(0.0f, 0.3f, 0.7f, 1.0f), 3);
647+}
libavfilter/opencl_source.h+1−0
@@ -23,6 +23,7 @@ extern const char *ff_opencl_source_avgblur; · 22 unmodified lines
2323 extern const char *ff_opencl_source_colorkey;
2424 extern const char *ff_opencl_source_colorspace_common;
2525 extern const char *ff_opencl_source_convolution;
26+extern const char *ff_opencl_source_deshake;
2627 extern const char *ff_opencl_source_neighbor;
2728 extern const char *ff_opencl_source_nlmeans;
2829 extern const char *ff_opencl_source_overlay;
libavfilter/version.h+1−1
@@ -31,7 +31,7 @@ · 30 unmodified lines
3131
3232 #define LIBAVFILTER_VERSION_MAJOR 7
3333 #define LIBAVFILTER_VERSION_MINOR 58
34−#define LIBAVFILTER_VERSION_MICRO 100
34+#define LIBAVFILTER_VERSION_MICRO 101
3535
3636
3737 #define LIBAVFILTER_VERSION_INT AV_VERSION_INT(LIBAVFILTER_VERSION_MAJOR, \
libavfilter/vf_deshake_opencl.c+2202−0
@@ -0,0 +1,2202 @@
1+/*
2+ * This file is part of FFmpeg.
3+ *
4+ * FFmpeg is free software; you can redistribute it and/or
5+ * modify it under the terms of the GNU Lesser General Public
6+ * License as published by the Free Software Foundation; either
7+ * version 2.1 of the License, or (at your option) any later version.
8+ *
9+ * FFmpeg is distributed in the hope that it will be useful,
10+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
11+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12+ * Lesser General Public License for more details.
13+ *
14+ * You should have received a copy of the GNU Lesser General Public
15+ * License along with FFmpeg; if not, write to the Free Software
16+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17+ *
18+ * Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
19+ * Copyright (C) 2009, Willow Garage Inc., all rights reserved.
20+ * Copyright (C) 2013, OpenCV Foundation, all rights reserved.
21+ * Third party copyrights are property of their respective owners.
22+ *
23+ * Redistribution and use in source and binary forms, with or without modification,
24+ * are permitted provided that the following conditions are met:
25+ *
26+ * * Redistribution's of source code must retain the above copyright notice,
27+ * this list of conditions and the following disclaimer.
28+ *
29+ * * Redistribution's in binary form must reproduce the above copyright notice,
30+ * this list of conditions and the following disclaimer in the documentation
31+ * and/or other materials provided with the distribution.
32+ *
33+ * * The name of the copyright holders may not be used to endorse or promote products
34+ * derived from this software without specific prior written permission.
35+ *
36+ * This software is provided by the copyright holders and contributors "as is" and
37+ * any express or implied warranties, including, but not limited to, the implied
38+ * warranties of merchantability and fitness for a particular purpose are disclaimed.
39+ * In no event shall the Intel Corporation or contributors be liable for any direct,
40+ * indirect, incidental, special, exemplary, or consequential damages
41+ * (including, but not limited to, procurement of substitute goods or services;
42+ * loss of use, data, or profits; or business interruption) however caused
43+ * and on any theory of liability, whether in contract, strict liability,
44+ * or tort (including negligence or otherwise) arising in any way out of
45+ * the use of this software, even if advised of the possibility of such damage.
46+ */
47+
48+#include <stdbool.h>
49+#include <float.h>
50+#include <libavutil/lfg.h>
51+#include "libavutil/opt.h"
52+#include "libavutil/imgutils.h"
53+#include "libavutil/mem.h"
54+#include "libavutil/fifo.h"
55+#include "libavutil/common.h"
56+#include "libavutil/avassert.h"
57+#include "libavutil/pixfmt.h"
58+#include "avfilter.h"
59+#include "framequeue.h"
60+#include "filters.h"
61+#include "transform.h"
62+#include "formats.h"
63+#include "internal.h"
64+#include "opencl.h"
65+#include "opencl_source.h"
66+#include "video.h"
67+
68+/*
69+This filter matches feature points between frames (dealing with outliers) and then
70+uses the matches to estimate an affine transform between frames. This transform is
71+decomposed into various values (translation, scale, rotation) and the values are
72+summed relative to the start of the video to obtain on absolute camera position
73+for each frame. This "camera path" is then smoothed via a gaussian filter, resulting
74+in a new path that is turned back into an affine transform and applied to each
75+frame to render it.
76+
77+High-level overview:
78+
79+All of the work to extract motion data from frames occurs in queue_frame. Motion data
80+is buffered in a smoothing window, so queue_frame simply computes the absolute camera
81+positions and places them in ringbuffers.
82+
83+filter_frame is responsible for looking at the absolute camera positions currently
84+in the ringbuffers, applying the gaussian filter, and then transforming the frames.
85+*/
86+
87+// Number of bits for BRIEF descriptors
88+#define BREIFN 512
89+// Size of the patch from which a BRIEF descriptor is extracted
90+// This is the size used in OpenCV
91+#define BRIEF_PATCH_SIZE 31
92+#define BRIEF_PATCH_SIZE_HALF (BRIEF_PATCH_SIZE / 2)
93+
94+#define MATCHES_CONTIG_SIZE 2000
95+
96+#define ROUNDED_UP_DIV(a, b) ((a + (b - 1)) / b)
97+
98+typedef struct PointPair {
99+ // Previous frame
100+ cl_float2 p1;
101+ // Current frame
102+ cl_float2 p2;
103+} PointPair;
104+
105+typedef struct MotionVector {
106+ PointPair p;
107+ // Used to mark vectors as potential outliers
108+ cl_int should_consider;
109+} MotionVector;
110+
111+// Denotes the indices for the different types of motion in the ringbuffers array
112+enum RingbufferIndices {
113+ RingbufX,
114+ RingbufY,
115+ RingbufRot,
116+ RingbufScaleX,
117+ RingbufScaleY,
118+
119+ // Should always be last
120+ RingbufCount
121+};
122+
123+// Struct that holds data for drawing point match debug data
124+typedef struct DebugMatches {
125+ MotionVector *matches;
126+ // The points used to calculate the affine transform for a frame
127+ MotionVector model_matches[3];
128+
129+ int num_matches;
130+ // For cases where we couldn't calculate a model
131+ int num_model_matches;
132+} DebugMatches;
133+
134+// Groups together the ringbuffers that store absolute distortion / position values
135+// for each frame
136+typedef struct AbsoluteFrameMotion {
137+ // Array with the various ringbuffers, indexed via the RingbufferIndices enum
138+ AVFifoBuffer *ringbuffers[RingbufCount];
139+
140+ // Offset to get to the current frame being processed
141+ // (not in bytes)
142+ int curr_frame_offset;
143+ // Keeps track of where the start and end of contiguous motion data is (to
144+ // deal with cases where no motion data is found between two frames)
145+ int data_start_offset;
146+ int data_end_offset;
147+
148+ AVFifoBuffer *debug_matches;
149+} AbsoluteFrameMotion;
150+
151+// Takes care of freeing the arrays within the DebugMatches inside of the
152+// debug_matches ringbuffer and then freeing the buffer itself.
153+static void free_debug_matches(AbsoluteFrameMotion *afm) {
154+ DebugMatches dm;
155+
156+ if (!afm->debug_matches) {
157+ return;
158+ }
159+
160+ while (av_fifo_size(afm->debug_matches) > 0) {
161+ av_fifo_generic_read(
162+ afm->debug_matches,
163+ &dm,
164+ sizeof(DebugMatches),
165+ NULL
166+ );
167+
168+ av_freep(&dm.matches);
169+ }
170+
171+ av_fifo_freep(&afm->debug_matches);
172+}
173+
174+// Stores the translation, scale, rotation, and skew deltas between two frames
175+typedef struct FrameDelta {
176+ cl_float2 translation;
177+ float rotation;
178+ cl_float2 scale;
179+ cl_float2 skew;
180+} FrameDelta;
181+
182+typedef struct SimilarityMatrix {
183+ // The 2x3 similarity matrix
184+ double matrix[6];
185+} SimilarityMatrix;
186+
187+typedef struct CropInfo {
188+ // The top left corner of the bounding box for the crop
189+ cl_float2 top_left;
190+ // The bottom right corner of the bounding box for the crop
191+ cl_float2 bottom_right;
192+} CropInfo;
193+
194+// Returned from function that determines start and end values for iteration
195+// around the current frame in a ringbuffer
196+typedef struct IterIndices {
197+ int start;
198+ int end;
199+} IterIndices;
200+
201+typedef struct DeshakeOpenCLContext {
202+ OpenCLFilterContext ocf;
203+ // Whether or not the above `OpenCLFilterContext` has been initialized
204+ int initialized;
205+
206+ // These variables are used in the activate callback
207+ int64_t duration;
208+ bool eof;
209+
210+ // State for random number generation
211+ AVLFG alfg;
212+
213+ // FIFO frame queue used to buffer future frames for processing
214+ FFFrameQueue fq;
215+ // Ringbuffers for frame positions
216+ AbsoluteFrameMotion abs_motion;
217+
218+ // The number of frames' motion to consider before and after the frame we are
219+ // smoothing
220+ int smooth_window;
221+ // The number of the frame we are currently processing
222+ int curr_frame;
223+
224+ // Stores a 1d array of normalised gaussian kernel values for convolution
225+ float *gauss_kernel;
226+
227+ // Buffer for error values used in RANSAC code
228+ float *ransac_err;
229+
230+ // Information regarding how to crop the smoothed luminance (or RGB) planes
231+ CropInfo crop_y;
232+ // Information regarding how to crop the smoothed chroma planes
233+ CropInfo crop_uv;
234+
235+ // Whether or not we are processing YUV input (as oppposed to RGB)
236+ bool is_yuv;
237+ // The underlying format of the hardware surfaces
238+ int sw_format;
239+
240+ // Buffer to copy `matches` into for the CPU to work with
241+ MotionVector *matches_host;
242+ MotionVector *matches_contig_host;
243+
244+ MotionVector *inliers;
245+
246+ cl_command_queue command_queue;
247+ cl_kernel kernel_grayscale;
248+ cl_kernel kernel_harris_response;
249+ cl_kernel kernel_refine_features;
250+ cl_kernel kernel_brief_descriptors;
251+ cl_kernel kernel_match_descriptors;
252+ cl_kernel kernel_transform;
253+ cl_kernel kernel_crop_upscale;
254+
255+ // Stores a frame converted to grayscale
256+ cl_mem grayscale;
257+ // Stores the harris response for a frame (measure of "cornerness" for each pixel)
258+ cl_mem harris_buf;
259+
260+ // Detected features after non-maximum suppression and sub-pixel refinement
261+ cl_mem refined_features;
262+ // Saved from the previous frame
263+ cl_mem prev_refined_features;
264+
265+ // BRIEF sampling pattern that is randomly initialized
266+ cl_mem brief_pattern;
267+ // Feature point descriptors for the current frame
268+ cl_mem descriptors;
269+ // Feature point descriptors for the previous frame
270+ cl_mem prev_descriptors;
271+ // Vectors between points in current and previous frame
272+ cl_mem matches;
273+ cl_mem matches_contig;
274+ // Holds the matrix to transform luminance (or RGB) with
275+ cl_mem transform_y;
276+ // Holds the matrix to transform chroma with
277+ cl_mem transform_uv;
278+
279+ // Configurable options
280+
281+ int tripod_mode;
282+ int debug_on;
283+ int should_crop;
284+
285+ // Whether or not feature points should be refined at a sub-pixel level
286+ cl_int refine_features;
287+ // If the user sets a value other than the default, 0, this percentage is
288+ // translated into a sigma value ranging from 0.5 to 40.0
289+ float smooth_percent;
290+ // This number is multiplied by the video frame rate to determine the size
291+ // of the smooth window
292+ float smooth_window_multiplier;
293+
294+ // Debug stuff
295+
296+ cl_kernel kernel_draw_debug_info;
297+ cl_mem debug_matches;
298+ cl_mem debug_model_matches;
299+
300+ // These store the total time spent executing the different kernels in nanoseconds
301+ unsigned long long grayscale_time;
302+ unsigned long long harris_response_time;
303+ unsigned long long refine_features_time;
304+ unsigned long long brief_descriptors_time;
305+ unsigned long long match_descriptors_time;
306+ unsigned long long transform_time;
307+ unsigned long long crop_upscale_time;
308+
309+ // Time spent copying matched features from the device to the host
310+ unsigned long long read_buf_time;
311+} DeshakeOpenCLContext;
312+
313+// Returns a random uniformly-distributed number in [low, high]
314+static int rand_in(int low, int high, AVLFG *alfg) {
315+ return (av_lfg_get(alfg) % (high - low)) + low;
316+}
317+
318+// Returns the average execution time for an event given the total time and the
319+// number of frames processed.
320+static double averaged_event_time_ms(unsigned long long total_time, int num_frames) {
321+ return (double)total_time / (double)num_frames / 1000000.0;
322+}
323+
324+// The following code is loosely ported from OpenCV
325+
326+// Estimates affine transform from 3 point pairs
327+// model is a 2x3 matrix:
328+// a b c
329+// d e f
330+static void run_estimate_kernel(const MotionVector *point_pairs, double *model)
331+{
332+ // src points
333+ double x1 = point_pairs[0].p.p1.s[0];
334+ double y1 = point_pairs[0].p.p1.s[1];
335+ double x2 = point_pairs[1].p.p1.s[0];
336+ double y2 = point_pairs[1].p.p1.s[1];
337+ double x3 = point_pairs[2].p.p1.s[0];
338+ double y3 = point_pairs[2].p.p1.s[1];
339+
340+ // dest points
341+ double X1 = point_pairs[0].p.p2.s[0];
342+ double Y1 = point_pairs[0].p.p2.s[1];
343+ double X2 = point_pairs[1].p.p2.s[0];
344+ double Y2 = point_pairs[1].p.p2.s[1];
345+ double X3 = point_pairs[2].p.p2.s[0];
346+ double Y3 = point_pairs[2].p.p2.s[1];
347+
348+ double d = 1.0 / ( x1*(y2-y3) + x2*(y3-y1) + x3*(y1-y2) );
349+
350+ model[0] = d * ( X1*(y2-y3) + X2*(y3-y1) + X3*(y1-y2) );
351+ model[1] = d * ( X1*(x3-x2) + X2*(x1-x3) + X3*(x2-x1) );
352+ model[2] = d * ( X1*(x2*y3 - x3*y2) + X2*(x3*y1 - x1*y3) + X3*(x1*y2 - x2*y1) );
353+
354+ model[3] = d * ( Y1*(y2-y3) + Y2*(y3-y1) + Y3*(y1-y2) );
355+ model[4] = d * ( Y1*(x3-x2) + Y2*(x1-x3) + Y3*(x2-x1) );
356+ model[5] = d * ( Y1*(x2*y3 - x3*y2) + Y2*(x3*y1 - x1*y3) + Y3*(x1*y2 - x2*y1) );
357+}
358+
359+// Checks that the 3 points in the given array are not collinear
360+static bool points_not_collinear(const cl_float2 **points)
361+{
362+ int j, k, i = 2;
363+
364+ for (j = 0; j < i; j++) {
365+ double dx1 = points[j]->s[0] - points[i]->s[0];
366+ double dy1 = points[j]->s[1] - points[i]->s[1];
367+
368+ for (k = 0; k < j; k++) {
369+ double dx2 = points[k]->s[0] - points[i]->s[0];
370+ double dy2 = points[k]->s[1] - points[i]->s[1];
371+
372+ // Assuming a 3840 x 2160 video with a point at (0, 0) and one at
373+ // (3839, 2159), this prevents a third point from being within roughly
374+ // 0.5 of a pixel of the line connecting the two on both axes
375+ if (fabs(dx2*dy1 - dy2*dx1) <= 1.0) {
376+ return false;
377+ }
378+ }
379+ }
380+
381+ return true;
382+}
383+
384+// Checks a subset of 3 point pairs to make sure that the points are not collinear
385+// and not too close to each other
386+static bool check_subset(const MotionVector *pairs_subset)
387+{
388+ const cl_float2 *prev_points[] = {
389+ &pairs_subset[0].p.p1,
390+ &pairs_subset[1].p.p1,
391+ &pairs_subset[2].p.p1
392+ };
393+
394+ const cl_float2 *curr_points[] = {
395+ &pairs_subset[0].p.p2,
396+ &pairs_subset[1].p.p2,
397+ &pairs_subset[2].p.p2
398+ };
399+
400+ return points_not_collinear(prev_points) && points_not_collinear(curr_points);
401+}
402+
403+// Selects a random subset of 3 points from point_pairs and places them in pairs_subset
404+static bool get_subset(
405+ AVLFG *alfg,
406+ const MotionVector *point_pairs,
407+ const int num_point_pairs,
408+ MotionVector *pairs_subset,
409+ int max_attempts
410+) {
411+ int idx[3];
412+ int i = 0, j, iters = 0;
413+
414+ for (; iters < max_attempts; iters++) {
415+ for (i = 0; i < 3 && iters < max_attempts;) {
416+ int idx_i = 0;
417+
418+ for (;;) {
419+ idx_i = idx[i] = rand_in(0, num_point_pairs, alfg);
420+
421+ for (j = 0; j < i; j++) {
422+ if (idx_i == idx[j]) {
423+ break;
424+ }
425+ }
426+
427+ if (j == i) {
428+ break;
429+ }
430+ }
431+
432+ pairs_subset[i] = point_pairs[idx[i]];
433+ i++;
434+ }
435+
436+ if (i == 3 && !check_subset(pairs_subset)) {
437+ continue;
438+ }
439+ break;
440+ }
441+
442+ return i == 3 && iters < max_attempts;
443+}
444+
445+// Computes the error for each of the given points based on the given model.
446+static void compute_error(
447+ const MotionVector *point_pairs,
448+ const int num_point_pairs,
449+ const double *model,
450+ float *err
451+) {
452+ double F0 = model[0], F1 = model[1], F2 = model[2];
453+ double F3 = model[3], F4 = model[4], F5 = model[5];
454+
455+ for (int i = 0; i < num_point_pairs; i++) {
456+ const cl_float2 *f = &point_pairs[i].p.p1;
457+ const cl_float2 *t = &point_pairs[i].p.p2;
458+
459+ double a = F0*f->s[0] + F1*f->s[1] + F2 - t->s[0];
460+ double b = F3*f->s[0] + F4*f->s[1] + F5 - t->s[1];
461+
462+ err[i] = a*a + b*b;
463+ }
464+}
465+
466+// Determines which of the given point matches are inliers for the given model
467+// based on the specified threshold.
468+//
469+// err must be an array of num_point_pairs length
470+static int find_inliers(
471+ MotionVector *point_pairs,
472+ const int num_point_pairs,
473+ const double *model,
474+ float *err,
475+ double thresh
476+) {
477+ float t = (float)(thresh * thresh);
478+ int i, n = num_point_pairs, num_inliers = 0;
479+
480+ compute_error(point_pairs, num_point_pairs, model, err);
481+
482+ for (i = 0; i < n; i++) {
483+ if (err[i] <= t) {
484+ // This is an inlier
485+ point_pairs[i].should_consider = true;
486+ num_inliers += 1;
487+ } else {
488+ point_pairs[i].should_consider = false;
489+ }
490+ }
491+
492+ return num_inliers;
493+}
494+
495+// Determines the number of iterations required to achieve the desired confidence level.
496+//
497+// The equation used to determine the number of iterations to do is:
498+// 1 - confidence = (1 - inlier_probability^num_points)^num_iters
499+//
500+// Solving for num_iters:
501+//
502+// num_iters = log(1 - confidence) / log(1 - inlier_probability^num_points)
503+//
504+// A more in-depth explanation can be found at https://en.wikipedia.org/wiki/Random_sample_consensus
505+// under the 'Parameters' heading
506+static int ransac_update_num_iters(double confidence, double num_outliers, int max_iters)
507+{
508+ double num, denom;
509+
510+ confidence = av_clipd(confidence, 0.0, 1.0);
511+ num_outliers = av_clipd(num_outliers, 0.0, 1.0);
512+
513+ // avoid inf's & nan's
514+ num = FFMAX(1.0 - confidence, DBL_MIN);
515+ denom = 1.0 - pow(1.0 - num_outliers, 3);
516+ if (denom < DBL_MIN) {
517+ return 0;
518+ }
519+
520+ num = log(num);
521+ denom = log(denom);
522+
523+ return denom >= 0 || -num >= max_iters * (-denom) ? max_iters : (int)round(num / denom);
524+}
525+
526+// Estimates an affine transform between the given pairs of points using RANdom
527+// SAmple Consensus
528+static bool estimate_affine_2d(
529+ DeshakeOpenCLContext *deshake_ctx,
530+ MotionVector *point_pairs,
531+ DebugMatches *debug_matches,
532+ const int num_point_pairs,
533+ double *model_out,
534+ const double threshold,
535+ const int max_iters,
536+ const double confidence
537+) {
538+ bool result = false;
539+ double best_model[6], model[6];
540+ MotionVector pairs_subset[3], best_pairs[3];
541+
542+ int iter, niters = FFMAX(max_iters, 1);
543+ int good_count, max_good_count = 0;
544+
545+ // We need at least 3 points to build a model from
546+ if (num_point_pairs < 3) {
547+ return false;
548+ } else if (num_point_pairs == 3) {
549+ // There are only 3 points, so RANSAC doesn't apply here
550+ run_estimate_kernel(point_pairs, model_out);
551+
552+ for (int i = 0; i < 3; ++i) {
553+ point_pairs[i].should_consider = true;
554+ }
555+
556+ return true;
557+ }
558+
559+ for (iter = 0; iter < niters; ++iter) {
560+ bool found = get_subset(&deshake_ctx->alfg, point_pairs, num_point_pairs, pairs_subset, 10000);
561+
562+ if (!found) {
563+ if (iter == 0) {
564+ return false;
565+ }
566+
567+ break;
568+ }
569+
570+ run_estimate_kernel(pairs_subset, model);
571+ good_count = find_inliers(point_pairs, num_point_pairs, model, deshake_ctx->ransac_err, threshold);
572+
573+ if (good_count > FFMAX(max_good_count, 2)) {
574+ for (int mi = 0; mi < 6; ++mi) {
575+ best_model[mi] = model[mi];
576+ }
577+
578+ for (int pi = 0; pi < 3; pi++) {
579+ best_pairs[pi] = pairs_subset[pi];
580+ }
581+
582+ max_good_count = good_count;
583+ niters = ransac_update_num_iters(
584+ confidence,
585+ (double)(num_point_pairs - good_count) / num_point_pairs,
586+ niters
587+ );
588+ }
589+ }
590+
591+ if (max_good_count > 0) {
592+ for (int mi = 0; mi < 6; ++mi) {
593+ model_out[mi] = best_model[mi];
594+ }
595+
596+ for (int pi = 0; pi < 3; ++pi) {
597+ debug_matches->model_matches[pi] = best_pairs[pi];
598+ }
599+ debug_matches->num_model_matches = 3;
600+
601+ // Find the inliers again for the best model for debugging
602+ find_inliers(point_pairs, num_point_pairs, best_model, deshake_ctx->ransac_err, threshold);
603+ result = true;
604+ }
605+
606+ return result;
607+}
608+
609+// "Wiggles" the first point in best_pairs around a tiny bit in order to decrease the
610+// total error
611+static void optimize_model(
612+ DeshakeOpenCLContext *deshake_ctx,
613+ MotionVector *best_pairs,
614+ MotionVector *inliers,
615+ const int num_inliers,
616+ float best_err,
617+ double *model_out
618+) {
619+ float move_x_val = 0.01;
620+ float move_y_val = 0.01;
621+ bool move_x = true;
622+ float old_move_x_val = 0;
623+ double model[6];
624+ int last_changed = 0;
625+
626+ for (int iters = 0; iters < 200; iters++) {
627+ float total_err = 0;
628+
629+ if (move_x) {
630+ best_pairs[0].p.p2.s[0] += move_x_val;
631+ } else {
632+ best_pairs[0].p.p2.s[0] += move_y_val;
633+ }
634+
635+ run_estimate_kernel(best_pairs, model);
636+ compute_error(inliers, num_inliers, model, deshake_ctx->ransac_err);
637+
638+ for (int j = 0; j < num_inliers; j++) {
639+ total_err += deshake_ctx->ransac_err[j];
640+ }
641+
642+ if (total_err < best_err) {
643+ for (int mi = 0; mi < 6; ++mi) {
644+ model_out[mi] = model[mi];
645+ }
646+
647+ best_err = total_err;
648+ last_changed = iters;
649+ } else {
650+ // Undo the change
651+ if (move_x) {
652+ best_pairs[0].p.p2.s[0] -= move_x_val;
653+ } else {
654+ best_pairs[0].p.p2.s[0] -= move_y_val;
655+ }
656+
657+ if (iters - last_changed > 4) {
658+ // We've already improved the model as much as we can
659+ break;
660+ }
661+
662+ old_move_x_val = move_x_val;
663+
664+ if (move_x) {
665+ move_x_val *= -1;
666+ } else {
667+ move_y_val *= -1;
668+ }
669+
670+ if (old_move_x_val < 0) {
671+ move_x = false;
672+ } else {
673+ move_x = true;
674+ }
675+ }
676+ }
677+}
678+
679+// Uses a process similar to that of RANSAC to find a transform that minimizes
680+// the total error for a set of point matches determined to be inliers
681+//
682+// (Pick random subsets, compute model, find total error, iterate until error
683+// is minimized.)
684+static bool minimize_error(
685+ DeshakeOpenCLContext *deshake_ctx,
686+ MotionVector *inliers,
687+ DebugMatches *debug_matches,
688+ const int num_inliers,
689+ double *model_out,
690+ const int max_iters
691+) {
692+ bool result = false;
693+ float best_err = FLT_MAX;
694+ double best_model[6], model[6];
695+ MotionVector pairs_subset[3], best_pairs[3];
696+
697+ for (int i = 0; i < max_iters; i++) {
698+ float total_err = 0;
699+ bool found = get_subset(&deshake_ctx->alfg, inliers, num_inliers, pairs_subset, 10000);
700+
701+ if (!found) {
702+ if (i == 0) {
703+ return false;
704+ }
705+
706+ break;
707+ }
708+
709+ run_estimate_kernel(pairs_subset, model);
710+ compute_error(inliers, num_inliers, model, deshake_ctx->ransac_err);
711+
712+ for (int j = 0; j < num_inliers; j++) {
713+ total_err += deshake_ctx->ransac_err[j];
714+ }
715+
716+ if (total_err < best_err) {
717+ for (int mi = 0; mi < 6; ++mi) {
718+ best_model[mi] = model[mi];
719+ }
720+
721+ for (int pi = 0; pi < 3; pi++) {
722+ best_pairs[pi] = pairs_subset[pi];
723+ }
724+
725+ best_err = total_err;
726+ }
727+ }
728+
729+ for (int mi = 0; mi < 6; ++mi) {
730+ model_out[mi] = best_model[mi];
731+ }
732+
733+ for (int pi = 0; pi < 3; ++pi) {
734+ debug_matches->model_matches[pi] = best_pairs[pi];
735+ }
736+ debug_matches->num_model_matches = 3;
737+ result = true;
738+
739+ optimize_model(deshake_ctx, best_pairs, inliers, num_inliers, best_err, model_out);
740+ return result;
741+}
742+
743+// End code from OpenCV
744+
745+// Decomposes a similarity matrix into translation, rotation, scale, and skew
746+//
747+// See http://frederic-wang.fr/decomposition-of-2d-transform-matrices.html
748+static FrameDelta decompose_transform(double *model)
749+{
750+ FrameDelta ret;
751+
752+ double a = model[0];
753+ double c = model[1];
754+ double e = model[2];
755+ double b = model[3];
756+ double d = model[4];
757+ double f = model[5];
758+ double delta = a * d - b * c;
759+
760+ ret.translation.s[0] = e;
761+ ret.translation.s[1] = f;
762+
763+ // This is the QR method
764+ if (a != 0 || b != 0) {
765+ double r = hypot(a, b);
766+
767+ ret.rotation = FFSIGN(b) * acos(a / r);
768+ ret.scale.s[0] = r;
769+ ret.scale.s[1] = delta / r;
770+ ret.skew.s[0] = atan((a * c + b * d) / (r * r));
771+ ret.skew.s[1] = 0;
772+ } else if (c != 0 || d != 0) {
773+ double s = sqrt(c * c + d * d);
774+
775+ ret.rotation = M_PI / 2 - FFSIGN(d) * acos(-c / s);
776+ ret.scale.s[0] = delta / s;
777+ ret.scale.s[1] = s;
778+ ret.skew.s[0] = 0;
779+ ret.skew.s[1] = atan((a * c + b * d) / (s * s));
780+ } // otherwise there is only translation
781+
782+ return ret;
783+}
784+
785+// Move valid vectors from the 2d buffer into a 1d buffer where they are contiguous
786+static int make_vectors_contig(
787+ DeshakeOpenCLContext *deshake_ctx,
788+ int size_y,
789+ int size_x
790+) {
791+ int num_vectors = 0;
792+
793+ for (int i = 0; i < size_y; ++i) {
794+ for (int j = 0; j < size_x; ++j) {
795+ MotionVector v = deshake_ctx->matches_host[j + i * size_x];
796+
797+ if (v.should_consider) {
798+ deshake_ctx->matches_contig_host[num_vectors] = v;
799+ ++num_vectors;
800+ }
801+
802+ // Make sure we do not exceed the amount of space we allocated for these vectors
803+ if (num_vectors == MATCHES_CONTIG_SIZE - 1) {
804+ return num_vectors;
805+ }
806+ }
807+ }
808+ return num_vectors;
809+}
810+
811+// Returns the gaussian kernel value for the given x coordinate and sigma value
812+static float gaussian_for(int x, float sigma) {
813+ return 1.0f / expf(((float)x * (float)x) / (2.0f * sigma * sigma));
814+}
815+
816+// Makes a normalized gaussian kernel of the given length for the given sigma
817+// and places it in gauss_kernel
818+static void make_gauss_kernel(float *gauss_kernel, float length, float sigma)
819+{
820+ float gauss_sum = 0;
821+ int window_half = length / 2;
822+
823+ for (int i = 0; i < length; ++i) {
824+ float val = gaussian_for(i - window_half, sigma);
825+
826+ gauss_sum += val;
827+ gauss_kernel[i] = val;
828+ }
829+
830+ // Normalize the gaussian values
831+ for (int i = 0; i < length; ++i) {
832+ gauss_kernel[i] /= gauss_sum;
833+ }
834+}
835+
836+// Returns indices to start and end iteration at in order to iterate over a window
837+// of length size centered at the current frame in a ringbuffer
838+//
839+// Always returns numbers that result in a window of length size, even if that
840+// means specifying negative indices or indices past the end of the values in the
841+// ringbuffers. Make sure you clip indices appropriately within your loop.
842+static IterIndices start_end_for(DeshakeOpenCLContext *deshake_ctx, int length) {
843+ IterIndices indices;
844+
845+ indices.start = deshake_ctx->abs_motion.curr_frame_offset - (length / 2);
846+ indices.end = deshake_ctx->abs_motion.curr_frame_offset + (length / 2) + (length % 2);
847+
848+ return indices;
849+}
850+
851+// Sets val to the value in the given ringbuffer at the given offset, taking care of
852+// clipping the offset into the appropriate range
853+static void ringbuf_float_at(
854+ DeshakeOpenCLContext *deshake_ctx,
855+ AVFifoBuffer *values,
856+ float *val,
857+ int offset
858+) {
859+ int clip_start, clip_end, offset_clipped;
860+ if (deshake_ctx->abs_motion.data_end_offset != -1) {
861+ clip_end = deshake_ctx->abs_motion.data_end_offset;
862+ } else {
863+ // This expression represents the last valid index in the buffer,
864+ // which we use repeatedly at the end of the video.
865+ clip_end = deshake_ctx->smooth_window - (av_fifo_space(values) / sizeof(float)) - 1;
866+ }
867+
868+ if (deshake_ctx->abs_motion.data_start_offset != -1) {
869+ clip_start = deshake_ctx->abs_motion.data_start_offset;
870+ } else {
871+ // Negative indices will occur at the start of the video, and we want
872+ // them to be clipped to 0 in order to repeatedly use the position of
873+ // the first frame.
874+ clip_start = 0;
875+ }
876+
877+ offset_clipped = av_clip(
878+ offset,
879+ clip_start,
880+ clip_end
881+ );
882+
883+ av_fifo_generic_peek_at(
884+ values,
885+ val,
886+ offset_clipped * sizeof(float),
887+ sizeof(float),
888+ NULL
889+ );
890+}
891+
892+// Returns smoothed current frame value of the given buffer of floats based on the
893+// given Gaussian kernel and its length (also the window length, centered around the
894+// current frame) and the "maximum value" of the motion.
895+//
896+// This "maximum value" should be the width / height of the image in the case of
897+// translation and an empirically chosen constant for rotation / scale.
898+//
899+// The sigma chosen to generate the final gaussian kernel with used to smooth the
900+// camera path is either hardcoded (set by user, deshake_ctx->smooth_percent) or
901+// adaptively chosen.
902+static float smooth(
903+ DeshakeOpenCLContext *deshake_ctx,
904+ float *gauss_kernel,
905+ int length,
906+ float max_val,
907+ AVFifoBuffer *values
908+) {
909+ float new_large_s = 0, new_small_s = 0, new_best = 0, old, diff_between,
910+ percent_of_max, inverted_percent;
911+ IterIndices indices = start_end_for(deshake_ctx, length);
912+ float large_sigma = 40.0f;
913+ float small_sigma = 2.0f;
914+ float best_sigma;
915+
916+ if (deshake_ctx->smooth_percent) {
917+ best_sigma = (large_sigma - 0.5f) * deshake_ctx->smooth_percent + 0.5f;
918+ } else {
919+ // Strategy to adaptively smooth trajectory:
920+ //
921+ // 1. Smooth path with large and small sigma values
922+ // 2. Take the absolute value of the difference between them
923+ // 3. Get a percentage by putting the difference over the "max value"
924+ // 4, Invert the percentage
925+ // 5. Calculate a new sigma value weighted towards the larger sigma value
926+ // 6. Determine final smoothed trajectory value using that sigma
927+
928+ make_gauss_kernel(gauss_kernel, length, large_sigma);
929+ for (int i = indices.start, j = 0; i < indices.end; ++i, ++j) {
930+ ringbuf_float_at(deshake_ctx, values, &old, i);
931+ new_large_s += old * gauss_kernel[j];
932+ }
933+
934+ make_gauss_kernel(gauss_kernel, length, small_sigma);
935+ for (int i = indices.start, j = 0; i < indices.end; ++i, ++j) {
936+ ringbuf_float_at(deshake_ctx, values, &old, i);
937+ new_small_s += old * gauss_kernel[j];
938+ }
939+
940+ diff_between = fabsf(new_large_s - new_small_s);
941+ percent_of_max = diff_between / max_val;
942+ inverted_percent = 1 - percent_of_max;
943+ best_sigma = large_sigma * powf(inverted_percent, 40);
944+ }
945+
946+ make_gauss_kernel(gauss_kernel, length, best_sigma);
947+ for (int i = indices.start, j = 0; i < indices.end; ++i, ++j) {
948+ ringbuf_float_at(deshake_ctx, values, &old, i);
949+ new_best += old * gauss_kernel[j];
950+ }
951+
952+ return new_best;
953+}
954+
955+// Returns the position of the given point after the transform is applied
956+static cl_float2 transformed_point(float x, float y, float *transform) {
957+ cl_float2 ret;
958+
959+ ret.s[0] = x * transform[0] + y * transform[1] + transform[2];
960+ ret.s[1] = x * transform[3] + y * transform[4] + transform[5];
961+
962+ return ret;
963+}
964+
965+// Creates an affine transform that scales from the center of a frame
966+static void transform_center_scale(
967+ float x_shift,
968+ float y_shift,
969+ float angle,
970+ float scale_x,
971+ float scale_y,
972+ float center_w,
973+ float center_h,
974+ float *matrix
975+) {
976+ cl_float2 center_s;
977+ float center_s_w, center_s_h;
978+
979+ ff_get_matrix(
980+ 0,
981+ 0,
982+ 0,
983+ scale_x,
984+ scale_y,
985+ matrix
986+ );
987+
988+ center_s = transformed_point(center_w, center_h, matrix);
989+ center_s_w = center_w - center_s.s[0];
990+ center_s_h = center_h - center_s.s[1];
991+
992+ ff_get_matrix(
993+ x_shift + center_s_w,
994+ y_shift + center_s_h,
995+ angle,
996+ scale_x,
997+ scale_y,
998+ matrix
999+ );
1000+}
1001+
1002+// Determines the crop necessary to eliminate black borders from a smoothed frame
1003+// and updates target crop accordingly
1004+static void update_needed_crop(
1005+ CropInfo* crop,
1006+ float *transform,
1007+ float frame_width,
1008+ float frame_height
1009+) {
1010+ float new_width, new_height, adjusted_width, adjusted_height, adjusted_x, adjusted_y;
1011+
1012+ cl_float2 top_left = transformed_point(0, 0, transform);
1013+ cl_float2 top_right = transformed_point(frame_width, 0, transform);
1014+ cl_float2 bottom_left = transformed_point(0, frame_height, transform);
1015+ cl_float2 bottom_right = transformed_point(frame_width, frame_height, transform);
1016+ float ar_h = frame_height / frame_width;
1017+ float ar_w = frame_width / frame_height;
1018+
1019+ if (crop->bottom_right.s[0] == 0) {
1020+ // The crop hasn't been set to the original size of the plane
1021+ crop->bottom_right.s[0] = frame_width;
1022+ crop->bottom_right.s[1] = frame_height;
1023+ }
1024+
1025+ crop->top_left.s[0] = FFMAX3(
1026+ crop->top_left.s[0],
1027+ top_left.s[0],
1028+ bottom_left.s[0]
1029+ );
1030+
1031+ crop->top_left.s[1] = FFMAX3(
1032+ crop->top_left.s[1],
1033+ top_left.s[1],
1034+ top_right.s[1]
1035+ );
1036+
1037+ crop->bottom_right.s[0] = FFMIN3(
1038+ crop->bottom_right.s[0],
1039+ bottom_right.s[0],
1040+ top_right.s[0]
1041+ );
1042+
1043+ crop->bottom_right.s[1] = FFMIN3(
1044+ crop->bottom_right.s[1],
1045+ bottom_right.s[1],
1046+ bottom_left.s[1]
1047+ );
1048+
1049+ // Make sure our potentially new bounding box has the same aspect ratio
1050+ new_height = crop->bottom_right.s[1] - crop->top_left.s[1];
1051+ new_width = crop->bottom_right.s[0] - crop->top_left.s[0];
1052+
1053+ adjusted_width = new_height * ar_w;
1054+ adjusted_x = crop->bottom_right.s[0] - adjusted_width;
1055+
1056+ if (adjusted_x >= crop->top_left.s[0]) {
1057+ crop->top_left.s[0] = adjusted_x;
1058+ } else {
1059+ adjusted_height = new_width * ar_h;
1060+ adjusted_y = crop->bottom_right.s[1] - adjusted_height;
1061+ crop->top_left.s[1] = adjusted_y;
1062+ }
1063+}
1064+
1065+static av_cold void deshake_opencl_uninit(AVFilterContext *avctx)
1066+{
1067+ DeshakeOpenCLContext *ctx = avctx->priv;
1068+ cl_int cle;
1069+
1070+ for (int i = 0; i < RingbufCount; i++)
1071+ av_fifo_freep(&ctx->abs_motion.ringbuffers[i]);
1072+
1073+ if (ctx->debug_on)
1074+ free_debug_matches(&ctx->abs_motion);
1075+
1076+ if (ctx->gauss_kernel)
1077+ av_freep(&ctx->gauss_kernel);
1078+
1079+ if (ctx->ransac_err)
1080+ av_freep(&ctx->ransac_err);
1081+
1082+ if (ctx->matches_host)
1083+ av_freep(&ctx->matches_host);
1084+
1085+ if (ctx->matches_contig_host)
1086+ av_freep(&ctx->matches_contig_host);
1087+
1088+ if (ctx->inliers)
1089+ av_freep(&ctx->inliers);
1090+
1091+ ff_framequeue_free(&ctx->fq);
1092+
1093+ CL_RELEASE_KERNEL(ctx->kernel_grayscale);
1094+ CL_RELEASE_KERNEL(ctx->kernel_harris_response);
1095+ CL_RELEASE_KERNEL(ctx->kernel_refine_features);
1096+ CL_RELEASE_KERNEL(ctx->kernel_brief_descriptors);
1097+ CL_RELEASE_KERNEL(ctx->kernel_match_descriptors);
1098+ CL_RELEASE_KERNEL(ctx->kernel_crop_upscale);
1099+ if (ctx->debug_on)
1100+ CL_RELEASE_KERNEL(ctx->kernel_draw_debug_info);
1101+
1102+ CL_RELEASE_QUEUE(ctx->command_queue);
1103+
1104+ if (!ctx->is_yuv)
1105+ CL_RELEASE_MEMORY(ctx->grayscale);
1106+ CL_RELEASE_MEMORY(ctx->harris_buf);
1107+ CL_RELEASE_MEMORY(ctx->refined_features);
1108+ CL_RELEASE_MEMORY(ctx->prev_refined_features);
1109+ CL_RELEASE_MEMORY(ctx->brief_pattern);
1110+ CL_RELEASE_MEMORY(ctx->descriptors);
1111+ CL_RELEASE_MEMORY(ctx->prev_descriptors);
1112+ CL_RELEASE_MEMORY(ctx->matches);
1113+ CL_RELEASE_MEMORY(ctx->matches_contig);
1114+ CL_RELEASE_MEMORY(ctx->transform_y);
1115+ CL_RELEASE_MEMORY(ctx->transform_uv);
1116+ if (ctx->debug_on) {
1117+ CL_RELEASE_MEMORY(ctx->debug_matches);
1118+ CL_RELEASE_MEMORY(ctx->debug_model_matches);
1119+ }
1120+
1121+ ff_opencl_filter_uninit(avctx);
1122+}
1123+
1124+static int deshake_opencl_init(AVFilterContext *avctx)
1125+{
1126+ DeshakeOpenCLContext *ctx = avctx->priv;
1127+ AVFilterLink *outlink = avctx->outputs[0];
1128+ AVFilterLink *inlink = avctx->inputs[0];
1129+ // Pointer to the host-side pattern buffer to be initialized and then copied
1130+ // to the GPU
1131+ PointPair *pattern_host;
1132+ cl_int cle;
1133+ int err;
1134+ cl_ulong8 zeroed_ulong8;
1135+ FFFrameQueueGlobal fqg;
1136+ cl_image_format grayscale_format;
1137+ cl_image_desc grayscale_desc;
1138+ cl_command_queue_properties queue_props;
1139+
1140+ const enum AVPixelFormat disallowed_formats[14] = {
1141+ AV_PIX_FMT_GBRP,
1142+ AV_PIX_FMT_GBRP9BE,
1143+ AV_PIX_FMT_GBRP9LE,
1144+ AV_PIX_FMT_GBRP10BE,
1145+ AV_PIX_FMT_GBRP10LE,
1146+ AV_PIX_FMT_GBRP16BE,
1147+ AV_PIX_FMT_GBRP16LE,
1148+ AV_PIX_FMT_GBRAP,
1149+ AV_PIX_FMT_GBRAP16BE,
1150+ AV_PIX_FMT_GBRAP16LE,
1151+ AV_PIX_FMT_GBRAP12BE,
1152+ AV_PIX_FMT_GBRAP12LE,
1153+ AV_PIX_FMT_GBRAP10BE,
1154+ AV_PIX_FMT_GBRAP10LE
1155+ };
1156+
1157+ // Number of elements for an array
1158+ const int image_grid_32 = ROUNDED_UP_DIV(outlink->h, 32) * ROUNDED_UP_DIV(outlink->w, 32);
1159+
1160+ const int descriptor_buf_size = image_grid_32 * (BREIFN / 8);
1161+ const int features_buf_size = image_grid_32 * sizeof(cl_float2);
1162+
1163+ const AVHWFramesContext *hw_frames_ctx = (AVHWFramesContext*)inlink->hw_frames_ctx->data;
1164+ const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(hw_frames_ctx->sw_format);
1165+
1166+ av_assert0(hw_frames_ctx);
1167+ av_assert0(desc);
1168+
1169+ ff_framequeue_global_init(&fqg);
1170+ ff_framequeue_init(&ctx->fq, &fqg);
1171+ ctx->eof = false;
1172+ ctx->smooth_window = (int)(av_q2d(avctx->inputs[0]->frame_rate) * ctx->smooth_window_multiplier);
1173+ ctx->curr_frame = 0;
1174+
1175+ memset(&zeroed_ulong8, 0, sizeof(cl_ulong8));
1176+
1177+ ctx->gauss_kernel = av_malloc_array(ctx->smooth_window, sizeof(float));
1178+ if (!ctx->gauss_kernel) {
1179+ err = AVERROR(ENOMEM);
1180+ goto fail;
1181+ }
1182+
1183+ ctx->ransac_err = av_malloc_array(MATCHES_CONTIG_SIZE, sizeof(float));
1184+ if (!ctx->ransac_err) {
1185+ err = AVERROR(ENOMEM);
1186+ goto fail;
1187+ }
1188+
1189+ for (int i = 0; i < RingbufCount; i++) {
1190+ ctx->abs_motion.ringbuffers[i] = av_fifo_alloc_array(
1191+ ctx->smooth_window,
1192+ sizeof(float)
1193+ );
1194+
1195+ if (!ctx->abs_motion.ringbuffers[i]) {
1196+ err = AVERROR(ENOMEM);
1197+ goto fail;
1198+ }
1199+ }
1200+
1201+ if (ctx->debug_on) {
1202+ ctx->abs_motion.debug_matches = av_fifo_alloc_array(
1203+ ctx->smooth_window / 2,
1204+ sizeof(DebugMatches)
1205+ );
1206+
1207+ if (!ctx->abs_motion.debug_matches) {
1208+ err = AVERROR(ENOMEM);
1209+ goto fail;
1210+ }
1211+ }
1212+
1213+ ctx->abs_motion.curr_frame_offset = 0;
1214+ ctx->abs_motion.data_start_offset = -1;
1215+ ctx->abs_motion.data_end_offset = -1;
1216+
1217+ pattern_host = av_malloc_array(BREIFN, sizeof(PointPair));
1218+ if (!pattern_host) {
1219+ err = AVERROR(ENOMEM);
1220+ goto fail;
1221+ }
1222+
1223+ ctx->matches_host = av_malloc_array(image_grid_32, sizeof(MotionVector));
1224+ if (!ctx->matches_host) {
1225+ err = AVERROR(ENOMEM);
1226+ goto fail;
1227+ }
1228+
1229+ ctx->matches_contig_host = av_malloc_array(MATCHES_CONTIG_SIZE, sizeof(MotionVector));
1230+ if (!ctx->matches_contig_host) {
1231+ err = AVERROR(ENOMEM);
1232+ goto fail;
1233+ }
1234+
1235+ ctx->inliers = av_malloc_array(MATCHES_CONTIG_SIZE, sizeof(MotionVector));
1236+ if (!ctx->inliers) {
1237+ err = AVERROR(ENOMEM);
1238+ goto fail;
1239+ }
1240+
1241+ // Initializing the patch pattern for building BREIF descriptors with
1242+ av_lfg_init(&ctx->alfg, 234342424);
1243+ for (int i = 0; i < BREIFN; ++i) {
1244+ PointPair pair;
1245+
1246+ for (int j = 0; j < 2; ++j) {
1247+ pair.p1.s[j] = rand_in(-BRIEF_PATCH_SIZE_HALF, BRIEF_PATCH_SIZE_HALF + 1, &ctx->alfg);
1248+ pair.p2.s[j] = rand_in(-BRIEF_PATCH_SIZE_HALF, BRIEF_PATCH_SIZE_HALF + 1, &ctx->alfg);
1249+ }
1250+
1251+ pattern_host[i] = pair;
1252+ }
1253+
1254+ for (int i = 0; i < 14; i++) {
1255+ if (ctx->sw_format == disallowed_formats[i]) {
1256+ av_log(avctx, AV_LOG_ERROR, "unsupported format in deshake_opencl.\n");
1257+ err = AVERROR(ENOSYS);
1258+ goto fail;
1259+ }
1260+ }
1261+
1262+ if (desc->flags & AV_PIX_FMT_FLAG_RGB) {
1263+ ctx->is_yuv = false;
1264+ } else {
1265+ ctx->is_yuv = true;
1266+ }
1267+ ctx->sw_format = hw_frames_ctx->sw_format;
1268+
1269+ err = ff_opencl_filter_load_program(avctx, &ff_opencl_source_deshake, 1);
1270+ if (err < 0)
1271+ goto fail;
1272+
1273+ if (ctx->debug_on) {
1274+ queue_props = CL_QUEUE_PROFILING_ENABLE;
1275+ } else {
1276+ queue_props = 0;
1277+ }
1278+ ctx->command_queue = clCreateCommandQueue(
1279+ ctx->ocf.hwctx->context,
1280+ ctx->ocf.hwctx->device_id,
1281+ queue_props,
1282+ &cle
1283+ );
1284+ CL_FAIL_ON_ERROR(AVERROR(EIO), "Failed to create OpenCL command queue %d.\n", cle);
1285+
1286+ CL_CREATE_KERNEL(ctx, grayscale);
1287+ CL_CREATE_KERNEL(ctx, harris_response);
1288+ CL_CREATE_KERNEL(ctx, refine_features);
1289+ CL_CREATE_KERNEL(ctx, brief_descriptors);
1290+ CL_CREATE_KERNEL(ctx, match_descriptors);
1291+ CL_CREATE_KERNEL(ctx, transform);
1292+ CL_CREATE_KERNEL(ctx, crop_upscale);
1293+ if (ctx->debug_on)
1294+ CL_CREATE_KERNEL(ctx, draw_debug_info);
1295+
1296+ if (!ctx->is_yuv) {
1297+ grayscale_format.image_channel_order = CL_R;
1298+ grayscale_format.image_channel_data_type = CL_FLOAT;
1299+
1300+ grayscale_desc = (cl_image_desc) {
1301+ .image_type = CL_MEM_OBJECT_IMAGE2D,
1302+ .image_width = outlink->w,
1303+ .image_height = outlink->h,
1304+ .image_depth = 0,
1305+ .image_array_size = 0,
1306+ .image_row_pitch = 0,
1307+ .image_slice_pitch = 0,
1308+ .num_mip_levels = 0,
1309+ .num_samples = 0,
1310+ .buffer = NULL,
1311+ };
1312+
1313+ ctx->grayscale = clCreateImage(
1314+ ctx->ocf.hwctx->context,
1315+ 0,
1316+ &grayscale_format,
1317+ &grayscale_desc,
1318+ NULL,
1319+ &cle
1320+ );
1321+ CL_FAIL_ON_ERROR(AVERROR(EIO), "Failed to create grayscale image: %d.\n", cle);
1322+ }
1323+
1324+ CL_CREATE_BUFFER(ctx, harris_buf, outlink->h * outlink->w * sizeof(float));
1325+ CL_CREATE_BUFFER(ctx, refined_features, features_buf_size);
1326+ CL_CREATE_BUFFER(ctx, prev_refined_features, features_buf_size);
1327+ CL_CREATE_BUFFER_FLAGS(
1328+ ctx,
1329+ brief_pattern,
1330+ CL_MEM_READ_WRITE | CL_MEM_COPY_HOST_PTR,
1331+ BREIFN * sizeof(PointPair),
1332+ pattern_host
1333+ );
1334+ CL_CREATE_BUFFER(ctx, descriptors, descriptor_buf_size);
1335+ CL_CREATE_BUFFER(ctx, prev_descriptors, descriptor_buf_size);
1336+ CL_CREATE_BUFFER(ctx, matches, image_grid_32 * sizeof(MotionVector));
1337+ CL_CREATE_BUFFER(ctx, matches_contig, MATCHES_CONTIG_SIZE * sizeof(MotionVector));
1338+ CL_CREATE_BUFFER(ctx, transform_y, 9 * sizeof(float));
1339+ CL_CREATE_BUFFER(ctx, transform_uv, 9 * sizeof(float));
1340+ if (ctx->debug_on) {
1341+ CL_CREATE_BUFFER(ctx, debug_matches, MATCHES_CONTIG_SIZE * sizeof(MotionVector));
1342+ CL_CREATE_BUFFER(ctx, debug_model_matches, 3 * sizeof(MotionVector));
1343+ }
1344+
1345+ ctx->initialized = 1;
1346+ av_freep(&pattern_host);
1347+
1348+ return 0;
1349+
1350+fail:
1351+ if (!pattern_host)
1352+ av_freep(&pattern_host);
1353+ return err;
1354+}
1355+
1356+// Logs debug information about the transform data
1357+static void transform_debug(AVFilterContext *avctx, float *new_vals, float *old_vals, int curr_frame) {
1358+ av_log(avctx, AV_LOG_VERBOSE,
1359+ "Frame %d:\n"
1360+ "\tframe moved from: %f x, %f y\n"
1361+ "\t to: %f x, %f y\n"
1362+ "\t rotated from: %f degrees\n"
1363+ "\t to: %f degrees\n"
1364+ "\t scaled from: %f x, %f y\n"
1365+ "\t to: %f x, %f y\n"
1366+ "\n"
1367+ "\tframe moved by: %f x, %f y\n"
1368+ "\t rotated by: %f degrees\n"
1369+ "\t scaled by: %f x, %f y\n",
1370+ curr_frame,
1371+ old_vals[RingbufX], old_vals[RingbufY],
1372+ new_vals[RingbufX], new_vals[RingbufY],
1373+ old_vals[RingbufRot] * (180.0 / M_PI),
1374+ new_vals[RingbufRot] * (180.0 / M_PI),
1375+ old_vals[RingbufScaleX], old_vals[RingbufScaleY],
1376+ new_vals[RingbufScaleX], new_vals[RingbufScaleY],
1377+ old_vals[RingbufX] - new_vals[RingbufX], old_vals[RingbufY] - new_vals[RingbufY],
1378+ old_vals[RingbufRot] * (180.0 / M_PI) - new_vals[RingbufRot] * (180.0 / M_PI),
1379+ new_vals[RingbufScaleX] / old_vals[RingbufScaleX], new_vals[RingbufScaleY] / old_vals[RingbufScaleY]
1380+ );
1381+}
1382+
1383+// Uses the buffered motion information to determine a transform that smooths the
1384+// given frame and applies it
1385+static int filter_frame(AVFilterLink *link, AVFrame *input_frame)
1386+{
1387+ AVFilterContext *avctx = link->dst;
1388+ AVFilterLink *outlink = avctx->outputs[0];
1389+ DeshakeOpenCLContext *deshake_ctx = avctx->priv;
1390+ AVFrame *cropped_frame = NULL, *transformed_frame = NULL;
1391+ int err;
1392+ cl_int cle;
1393+ float new_vals[RingbufCount];
1394+ float old_vals[RingbufCount];
1395+ // Luma (in the case of YUV) transform, or just the transform in the case of RGB
1396+ float transform_y[9];
1397+ // Chroma transform
1398+ float transform_uv[9];
1399+ // Luma crop transform (or RGB)
1400+ float transform_crop_y[9];
1401+ // Chroma crop transform
1402+ float transform_crop_uv[9];
1403+ float transform_debug_rgb[9];
1404+ size_t global_work[2];
1405+ int64_t duration;
1406+ cl_mem src, transformed, dst;
1407+ cl_mem transforms[3];
1408+ CropInfo crops[3];
1409+ cl_event transform_event, crop_upscale_event;
1410+ DebugMatches debug_matches;
1411+ cl_int num_model_matches;
1412+
1413+ const float center_w = (float)input_frame->width / 2;
1414+ const float center_h = (float)input_frame->height / 2;
1415+
1416+ const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(deshake_ctx->sw_format);
1417+ const int chroma_width = AV_CEIL_RSHIFT(input_frame->width, desc->log2_chroma_w);
1418+ const int chroma_height = AV_CEIL_RSHIFT(input_frame->height, desc->log2_chroma_h);
1419+
1420+ const float center_w_chroma = (float)chroma_width / 2;
1421+ const float center_h_chroma = (float)chroma_height / 2;
1422+
1423+ const float luma_w_over_chroma_w = ((float)input_frame->width / (float)chroma_width);
1424+ const float luma_h_over_chroma_h = ((float)input_frame->height / (float)chroma_height);
1425+
1426+ if (deshake_ctx->debug_on) {
1427+ av_fifo_generic_read(
1428+ deshake_ctx->abs_motion.debug_matches,
1429+ &debug_matches,
1430+ sizeof(DebugMatches),
1431+ NULL
1432+ );
1433+ }
1434+
1435+ if (input_frame->pkt_duration) {
1436+ duration = input_frame->pkt_duration;
1437+ } else {
1438+ duration = av_rescale_q(1, av_inv_q(outlink->frame_rate), outlink->time_base);
1439+ }
1440+ deshake_ctx->duration = input_frame->pts + duration;
1441+
1442+ // Get the absolute transform data for this frame
1443+ for (int i = 0; i < RingbufCount; i++) {
1444+ av_fifo_generic_peek_at(
1445+ deshake_ctx->abs_motion.ringbuffers[i],
1446+ &old_vals[i],
1447+ deshake_ctx->abs_motion.curr_frame_offset * sizeof(float),
1448+ sizeof(float),
1449+ NULL
1450+ );
1451+ }
1452+
1453+ if (deshake_ctx->tripod_mode) {
1454+ // If tripod mode is turned on we simply undo all motion relative to the
1455+ // first frame
1456+
1457+ new_vals[RingbufX] = 0.0f;
1458+ new_vals[RingbufY] = 0.0f;
1459+ new_vals[RingbufRot] = 0.0f;
1460+ new_vals[RingbufScaleX] = 1.0f;
1461+ new_vals[RingbufScaleY] = 1.0f;
1462+ } else {
1463+ // Tripod mode is off and we need to smooth a moving camera
1464+
1465+ new_vals[RingbufX] = smooth(
1466+ deshake_ctx,
1467+ deshake_ctx->gauss_kernel,
1468+ deshake_ctx->smooth_window,
1469+ input_frame->width,
1470+ deshake_ctx->abs_motion.ringbuffers[RingbufX]
1471+ );
1472+ new_vals[RingbufY] = smooth(
1473+ deshake_ctx,
1474+ deshake_ctx->gauss_kernel,
1475+ deshake_ctx->smooth_window,
1476+ input_frame->height,
1477+ deshake_ctx->abs_motion.ringbuffers[RingbufY]
1478+ );
1479+ new_vals[RingbufRot] = smooth(
1480+ deshake_ctx,
1481+ deshake_ctx->gauss_kernel,
1482+ deshake_ctx->smooth_window,
1483+ M_PI / 4,
1484+ deshake_ctx->abs_motion.ringbuffers[RingbufRot]
1485+ );
1486+ new_vals[RingbufScaleX] = smooth(
1487+ deshake_ctx,
1488+ deshake_ctx->gauss_kernel,
1489+ deshake_ctx->smooth_window,
1490+ 2.0f,
1491+ deshake_ctx->abs_motion.ringbuffers[RingbufScaleX]
1492+ );
1493+ new_vals[RingbufScaleY] = smooth(
1494+ deshake_ctx,
1495+ deshake_ctx->gauss_kernel,
1496+ deshake_ctx->smooth_window,
1497+ 2.0f,
1498+ deshake_ctx->abs_motion.ringbuffers[RingbufScaleY]
1499+ );
1500+ }
1501+
1502+ transform_center_scale(
1503+ old_vals[RingbufX] - new_vals[RingbufX],
1504+ old_vals[RingbufY] - new_vals[RingbufY],
1505+ old_vals[RingbufRot] - new_vals[RingbufRot],
1506+ new_vals[RingbufScaleX] / old_vals[RingbufScaleX],
1507+ new_vals[RingbufScaleY] / old_vals[RingbufScaleY],
1508+ center_w,
1509+ center_h,
1510+ transform_y
1511+ );
1512+
1513+ transform_center_scale(
1514+ (old_vals[RingbufX] - new_vals[RingbufX]) / luma_w_over_chroma_w,
1515+ (old_vals[RingbufY] - new_vals[RingbufY]) / luma_h_over_chroma_h,
1516+ old_vals[RingbufRot] - new_vals[RingbufRot],
1517+ new_vals[RingbufScaleX] / old_vals[RingbufScaleX],
1518+ new_vals[RingbufScaleY] / old_vals[RingbufScaleY],
1519+ center_w_chroma,
1520+ center_h_chroma,
1521+ transform_uv
1522+ );
1523+
1524+ CL_BLOCKING_WRITE_BUFFER(deshake_ctx->command_queue, deshake_ctx->transform_y, 9 * sizeof(float), transform_y, NULL);
1525+ CL_BLOCKING_WRITE_BUFFER(deshake_ctx->command_queue, deshake_ctx->transform_uv, 9 * sizeof(float), transform_uv, NULL);
1526+
1527+ if (deshake_ctx->debug_on)
1528+ transform_debug(avctx, new_vals, old_vals, deshake_ctx->curr_frame);
1529+
1530+ cropped_frame = ff_get_video_buffer(outlink, outlink->w, outlink->h);
1531+ if (!cropped_frame) {
1532+ err = AVERROR(ENOMEM);
1533+ goto fail;
1534+ }
1535+
1536+ transformed_frame = ff_get_video_buffer(outlink, outlink->w, outlink->h);
1537+ if (!transformed_frame) {
1538+ err = AVERROR(ENOMEM);
1539+ goto fail;
1540+ }
1541+
1542+ transforms[0] = deshake_ctx->transform_y;
1543+ transforms[1] = transforms[2] = deshake_ctx->transform_uv;
1544+
1545+ for (int p = 0; p < FF_ARRAY_ELEMS(transformed_frame->data); p++) {
1546+ // Transform all of the planes appropriately
1547+ src = (cl_mem)input_frame->data[p];
1548+ transformed = (cl_mem)transformed_frame->data[p];
1549+
1550+ if (!transformed)
1551+ break;
1552+
1553+ err = ff_opencl_filter_work_size_from_image(avctx, global_work, input_frame, p, 0);
1554+ if (err < 0)
1555+ goto fail;
1556+
1557+ CL_RUN_KERNEL_WITH_ARGS(
1558+ deshake_ctx->command_queue,
1559+ deshake_ctx->kernel_transform,
1560+ global_work,
1561+ NULL,
1562+ &transform_event,
1563+ { sizeof(cl_mem), &src },
1564+ { sizeof(cl_mem), &transformed },
1565+ { sizeof(cl_mem), &transforms[p] },
1566+ );
1567+ }
1568+
1569+ if (deshake_ctx->debug_on && !deshake_ctx->is_yuv && debug_matches.num_matches > 0) {
1570+ CL_BLOCKING_WRITE_BUFFER(
1571+ deshake_ctx->command_queue,
1572+ deshake_ctx->debug_matches,
1573+ debug_matches.num_matches * sizeof(MotionVector),
1574+ debug_matches.matches,
1575+ NULL
1576+ );
1577+
1578+ CL_BLOCKING_WRITE_BUFFER(
1579+ deshake_ctx->command_queue,
1580+ deshake_ctx->debug_model_matches,
1581+ debug_matches.num_model_matches * sizeof(MotionVector),
1582+ debug_matches.model_matches,
1583+ NULL
1584+ );
1585+
1586+ num_model_matches = debug_matches.num_model_matches;
1587+
1588+ // Invert the transform
1589+ transform_center_scale(
1590+ new_vals[RingbufX] - old_vals[RingbufX],
1591+ new_vals[RingbufY] - old_vals[RingbufY],
1592+ new_vals[RingbufRot] - old_vals[RingbufRot],
1593+ old_vals[RingbufScaleX] / new_vals[RingbufScaleX],
1594+ old_vals[RingbufScaleY] / new_vals[RingbufScaleY],
1595+ center_w,
1596+ center_h,
1597+ transform_debug_rgb
1598+ );
1599+
1600+ CL_BLOCKING_WRITE_BUFFER(deshake_ctx->command_queue, deshake_ctx->transform_y, 9 * sizeof(float), transform_debug_rgb, NULL);
1601+
1602+ transformed = (cl_mem)transformed_frame->data[0];
1603+ CL_RUN_KERNEL_WITH_ARGS(
1604+ deshake_ctx->command_queue,
1605+ deshake_ctx->kernel_draw_debug_info,
1606+ (size_t[]){ debug_matches.num_matches },
1607+ NULL,
1608+ NULL,
1609+ { sizeof(cl_mem), &transformed },
1610+ { sizeof(cl_mem), &deshake_ctx->debug_matches },
1611+ { sizeof(cl_mem), &deshake_ctx->debug_model_matches },
1612+ { sizeof(cl_int), &num_model_matches },
1613+ { sizeof(cl_mem), &deshake_ctx->transform_y }
1614+ );
1615+ }
1616+
1617+ if (deshake_ctx->should_crop) {
1618+ // Generate transforms for cropping
1619+ transform_center_scale(
1620+ (old_vals[RingbufX] - new_vals[RingbufX]) / 5,
1621+ (old_vals[RingbufY] - new_vals[RingbufY]) / 5,
1622+ (old_vals[RingbufRot] - new_vals[RingbufRot]) / 5,
1623+ new_vals[RingbufScaleX] / old_vals[RingbufScaleX],
1624+ new_vals[RingbufScaleY] / old_vals[RingbufScaleY],
1625+ center_w,
1626+ center_h,
1627+ transform_crop_y
1628+ );
1629+ update_needed_crop(&deshake_ctx->crop_y, transform_crop_y, input_frame->width, input_frame->height);
1630+
1631+ transform_center_scale(
1632+ (old_vals[RingbufX] - new_vals[RingbufX]) / (5 * luma_w_over_chroma_w),
1633+ (old_vals[RingbufY] - new_vals[RingbufY]) / (5 * luma_h_over_chroma_h),
1634+ (old_vals[RingbufRot] - new_vals[RingbufRot]) / 5,
1635+ new_vals[RingbufScaleX] / old_vals[RingbufScaleX],
1636+ new_vals[RingbufScaleY] / old_vals[RingbufScaleY],
1637+ center_w_chroma,
1638+ center_h_chroma,
1639+ transform_crop_uv
1640+ );
1641+ update_needed_crop(&deshake_ctx->crop_uv, transform_crop_uv, chroma_width, chroma_height);
1642+
1643+ crops[0] = deshake_ctx->crop_y;
1644+ crops[1] = crops[2] = deshake_ctx->crop_uv;
1645+
1646+ for (int p = 0; p < FF_ARRAY_ELEMS(cropped_frame->data); p++) {
1647+ // Crop all of the planes appropriately
1648+ dst = (cl_mem)cropped_frame->data[p];
1649+ transformed = (cl_mem)transformed_frame->data[p];
1650+
1651+ if (!dst)
1652+ break;
1653+
1654+ err = ff_opencl_filter_work_size_from_image(avctx, global_work, input_frame, p, 0);
1655+ if (err < 0)
1656+ goto fail;
1657+
1658+ CL_RUN_KERNEL_WITH_ARGS(
1659+ deshake_ctx->command_queue,
1660+ deshake_ctx->kernel_crop_upscale,
1661+ global_work,
1662+ NULL,
1663+ &crop_upscale_event,
1664+ { sizeof(cl_mem), &transformed },
1665+ { sizeof(cl_mem), &dst },
1666+ { sizeof(cl_float2), &crops[p].top_left },
1667+ { sizeof(cl_float2), &crops[p].bottom_right },
1668+ );
1669+ }
1670+ }
1671+
1672+ if (deshake_ctx->curr_frame < deshake_ctx->smooth_window / 2) {
1673+ // This means we are somewhere at the start of the video. We need to
1674+ // increment the current frame offset until it reaches the center of
1675+ // the ringbuffers (as the current frame will be located there for
1676+ // the rest of the video).
1677+ //
1678+ // The end of the video is taken care of by draining motion data
1679+ // one-by-one out of the buffer, causing the (at that point fixed)
1680+ // offset to move towards later frames' data.
1681+ ++deshake_ctx->abs_motion.curr_frame_offset;
1682+ }
1683+
1684+ if (deshake_ctx->abs_motion.data_end_offset != -1) {
1685+ // Keep the end offset in sync with the frame it's supposed to be
1686+ // positioned at
1687+ --deshake_ctx->abs_motion.data_end_offset;
1688+
1689+ if (deshake_ctx->abs_motion.data_end_offset == deshake_ctx->abs_motion.curr_frame_offset - 1) {
1690+ // The end offset would be the start of the new video sequence; flip to
1691+ // start offset
1692+ deshake_ctx->abs_motion.data_end_offset = -1;
1693+ deshake_ctx->abs_motion.data_start_offset = deshake_ctx->abs_motion.curr_frame_offset;
1694+ }
1695+ } else if (deshake_ctx->abs_motion.data_start_offset != -1) {
1696+ // Keep the start offset in sync with the frame it's supposed to be
1697+ // positioned at
1698+ --deshake_ctx->abs_motion.data_start_offset;
1699+ }
1700+
1701+ if (deshake_ctx->debug_on) {
1702+ deshake_ctx->transform_time += ff_opencl_get_event_time(transform_event);
1703+ if (deshake_ctx->should_crop) {
1704+ deshake_ctx->crop_upscale_time += ff_opencl_get_event_time(crop_upscale_event);
1705+ }
1706+ }
1707+
1708+ ++deshake_ctx->curr_frame;
1709+
1710+ if (deshake_ctx->debug_on)
1711+ av_freep(&debug_matches.matches);
1712+
1713+ if (deshake_ctx->should_crop) {
1714+ err = av_frame_copy_props(cropped_frame, input_frame);
1715+ if (err < 0)
1716+ goto fail;
1717+
1718+ av_frame_free(&transformed_frame);
1719+ av_frame_free(&input_frame);
1720+ return ff_filter_frame(outlink, cropped_frame);
1721+
1722+ } else {
1723+ err = av_frame_copy_props(transformed_frame, input_frame);
1724+ if (err < 0)
1725+ goto fail;
1726+
1727+ av_frame_free(&cropped_frame);
1728+ av_frame_free(&input_frame);
1729+ return ff_filter_frame(outlink, transformed_frame);
1730+ }
1731+
1732+fail:
1733+ clFinish(deshake_ctx->command_queue);
1734+
1735+ if (deshake_ctx->debug_on)
1736+ if (debug_matches.matches)
1737+ av_freep(&debug_matches.matches);
1738+
1739+ av_frame_free(&input_frame);
1740+ av_frame_free(&transformed_frame);
1741+ av_frame_free(&cropped_frame);
1742+ return err;
1743+}
1744+
1745+// Add the given frame to the frame queue to eventually be processed.
1746+//
1747+// Also determines the motion from the previous frame and updates the stored
1748+// motion information accordingly.
1749+static int queue_frame(AVFilterLink *link, AVFrame *input_frame)
1750+{
1751+ AVFilterContext *avctx = link->dst;
1752+ DeshakeOpenCLContext *deshake_ctx = avctx->priv;
1753+ int err;
1754+ int num_vectors;
1755+ int num_inliers = 0;
1756+ cl_int cle;
1757+ FrameDelta relative;
1758+ SimilarityMatrix model;
1759+ size_t global_work[2];
1760+ size_t harris_global_work[2];
1761+ size_t grid_32_global_work[2];
1762+ int grid_32_h, grid_32_w;
1763+ size_t local_work[2];
1764+ cl_mem src, temp;
1765+ float prev_vals[5];
1766+ float new_vals[5];
1767+ cl_event grayscale_event, harris_response_event, refine_features_event,
1768+ brief_event, match_descriptors_event, read_buf_event;
1769+ DebugMatches debug_matches;
1770+
1771+ num_vectors = 0;
1772+
1773+ local_work[0] = 8;
1774+ local_work[1] = 8;
1775+
1776+ err = ff_opencl_filter_work_size_from_image(avctx, global_work, input_frame, 0, 0);
1777+ if (err < 0)
1778+ goto fail;
1779+
1780+ err = ff_opencl_filter_work_size_from_image(avctx, harris_global_work, input_frame, 0, 8);
1781+ if (err < 0)
1782+ goto fail;
1783+
1784+ err = ff_opencl_filter_work_size_from_image(avctx, grid_32_global_work, input_frame, 0, 32);
1785+ if (err < 0)
1786+ goto fail;
1787+
1788+ // We want a single work-item for each 32x32 block of pixels in the input frame
1789+ grid_32_global_work[0] /= 32;
1790+ grid_32_global_work[1] /= 32;
1791+
1792+ grid_32_h = ROUNDED_UP_DIV(input_frame->height, 32);
1793+ grid_32_w = ROUNDED_UP_DIV(input_frame->width, 32);
1794+
1795+ if (deshake_ctx->is_yuv) {
1796+ deshake_ctx->grayscale = (cl_mem)input_frame->data[0];
1797+ } else {
1798+ src = (cl_mem)input_frame->data[0];
1799+
1800+ CL_RUN_KERNEL_WITH_ARGS(
1801+ deshake_ctx->command_queue,
1802+ deshake_ctx->kernel_grayscale,
1803+ global_work,
1804+ NULL,
1805+ &grayscale_event,
1806+ { sizeof(cl_mem), &src },
1807+ { sizeof(cl_mem), &deshake_ctx->grayscale }
1808+ );
1809+ }
1810+
1811+ CL_RUN_KERNEL_WITH_ARGS(
1812+ deshake_ctx->command_queue,
1813+ deshake_ctx->kernel_harris_response,
1814+ harris_global_work,
1815+ local_work,
1816+ &harris_response_event,
1817+ { sizeof(cl_mem), &deshake_ctx->grayscale },
1818+ { sizeof(cl_mem), &deshake_ctx->harris_buf }
1819+ );
1820+
1821+ CL_RUN_KERNEL_WITH_ARGS(
1822+ deshake_ctx->command_queue,
1823+ deshake_ctx->kernel_refine_features,
1824+ grid_32_global_work,
1825+ NULL,
1826+ &refine_features_event,
1827+ { sizeof(cl_mem), &deshake_ctx->grayscale },
1828+ { sizeof(cl_mem), &deshake_ctx->harris_buf },
1829+ { sizeof(cl_mem), &deshake_ctx->refined_features },
1830+ { sizeof(cl_int), &deshake_ctx->refine_features }
1831+ );
1832+
1833+ CL_RUN_KERNEL_WITH_ARGS(
1834+ deshake_ctx->command_queue,
1835+ deshake_ctx->kernel_brief_descriptors,
1836+ grid_32_global_work,
1837+ NULL,
1838+ &brief_event,
1839+ { sizeof(cl_mem), &deshake_ctx->grayscale },
1840+ { sizeof(cl_mem), &deshake_ctx->refined_features },
1841+ { sizeof(cl_mem), &deshake_ctx->descriptors },
1842+ { sizeof(cl_mem), &deshake_ctx->brief_pattern}
1843+ );
1844+
1845+ if (av_fifo_size(deshake_ctx->abs_motion.ringbuffers[RingbufX]) == 0) {
1846+ // This is the first frame we've been given to queue, meaning there is
1847+ // no previous frame to match descriptors to
1848+
1849+ goto no_motion_data;
1850+ }
1851+
1852+ CL_RUN_KERNEL_WITH_ARGS(
1853+ deshake_ctx->command_queue,
1854+ deshake_ctx->kernel_match_descriptors,
1855+ grid_32_global_work,
1856+ NULL,
1857+ &match_descriptors_event,
1858+ { sizeof(cl_mem), &deshake_ctx->prev_refined_features },
1859+ { sizeof(cl_mem), &deshake_ctx->refined_features },
1860+ { sizeof(cl_mem), &deshake_ctx->descriptors },
1861+ { sizeof(cl_mem), &deshake_ctx->prev_descriptors },
1862+ { sizeof(cl_mem), &deshake_ctx->matches }
1863+ );
1864+
1865+ cle = clEnqueueReadBuffer(
1866+ deshake_ctx->command_queue,
1867+ deshake_ctx->matches,
1868+ CL_TRUE,
1869+ 0,
1870+ grid_32_h * grid_32_w * sizeof(MotionVector),
1871+ deshake_ctx->matches_host,
1872+ 0,
1873+ NULL,
1874+ &read_buf_event
1875+ );
1876+ CL_FAIL_ON_ERROR(AVERROR(EIO), "Failed to read matches to host: %d.\n", cle);
1877+
1878+ num_vectors = make_vectors_contig(deshake_ctx, grid_32_h, grid_32_w);
1879+
1880+ if (num_vectors < 10) {
1881+ // Not enough matches to get reliable motion data for this frame
1882+ //
1883+ // From this point on all data is relative to this frame rather than the
1884+ // original frame. We have to make sure that we don't mix values that were
1885+ // relative to the original frame with the new values relative to this
1886+ // frame when doing the gaussian smoothing. We keep track of where the old
1887+ // values end using this data_end_offset field in order to accomplish
1888+ // that goal.
1889+ //
1890+ // If no motion data is present for multiple frames in a short window of
1891+ // time, we leave the end where it was to avoid mixing 0s in with the
1892+ // old data (and just treat them all as part of the new values)
1893+ if (deshake_ctx->abs_motion.data_end_offset == -1) {
1894+ deshake_ctx->abs_motion.data_end_offset =
1895+ av_fifo_size(deshake_ctx->abs_motion.ringbuffers[RingbufX]) / sizeof(float) - 1;
1896+ }
1897+
1898+ goto no_motion_data;
1899+ }
1900+
1901+ if (!estimate_affine_2d(
1902+ deshake_ctx,
1903+ deshake_ctx->matches_contig_host,
1904+ &debug_matches,
1905+ num_vectors,
1906+ model.matrix,
1907+ 10.0,
1908+ 3000,
1909+ 0.999999999999
1910+ )) {
1911+ goto no_motion_data;
1912+ }
1913+
1914+ for (int i = 0; i < num_vectors; i++) {
1915+ if (deshake_ctx->matches_contig_host[i].should_consider) {
1916+ deshake_ctx->inliers[num_inliers] = deshake_ctx->matches_contig_host[i];
1917+ num_inliers++;
1918+ }
1919+ }
1920+
1921+ if (!minimize_error(
1922+ deshake_ctx,
1923+ deshake_ctx->inliers,
1924+ &debug_matches,
1925+ num_inliers,
1926+ model.matrix,
1927+ 400
1928+ )) {
1929+ goto no_motion_data;
1930+ }
1931+
1932+
1933+ relative = decompose_transform(model.matrix);
1934+
1935+ // Get the absolute transform data for the previous frame
1936+ for (int i = 0; i < RingbufCount; i++) {
1937+ av_fifo_generic_peek_at(
1938+ deshake_ctx->abs_motion.ringbuffers[i],
1939+ &prev_vals[i],
1940+ av_fifo_size(deshake_ctx->abs_motion.ringbuffers[i]) - sizeof(float),
1941+ sizeof(float),
1942+ NULL
1943+ );
1944+ }
1945+
1946+ new_vals[RingbufX] = prev_vals[RingbufX] + relative.translation.s[0];
1947+ new_vals[RingbufY] = prev_vals[RingbufY] + relative.translation.s[1];
1948+ new_vals[RingbufRot] = prev_vals[RingbufRot] + relative.rotation;
1949+ new_vals[RingbufScaleX] = prev_vals[RingbufScaleX] / relative.scale.s[0];
1950+ new_vals[RingbufScaleY] = prev_vals[RingbufScaleY] / relative.scale.s[1];
1951+
1952+ if (deshake_ctx->debug_on) {
1953+ if (!deshake_ctx->is_yuv) {
1954+ deshake_ctx->grayscale_time += ff_opencl_get_event_time(grayscale_event);
1955+ }
1956+ deshake_ctx->harris_response_time += ff_opencl_get_event_time(harris_response_event);
1957+ deshake_ctx->refine_features_time += ff_opencl_get_event_time(refine_features_event);
1958+ deshake_ctx->brief_descriptors_time += ff_opencl_get_event_time(brief_event);
1959+ deshake_ctx->match_descriptors_time += ff_opencl_get_event_time(match_descriptors_event);
1960+ deshake_ctx->read_buf_time += ff_opencl_get_event_time(read_buf_event);
1961+ }
1962+
1963+ goto end;
1964+
1965+no_motion_data:
1966+ new_vals[RingbufX] = 0.0f;
1967+ new_vals[RingbufY] = 0.0f;
1968+ new_vals[RingbufRot] = 0.0f;
1969+ new_vals[RingbufScaleX] = 1.0f;
1970+ new_vals[RingbufScaleY] = 1.0f;
1971+
1972+ for (int i = 0; i < num_vectors; i++) {
1973+ deshake_ctx->matches_contig_host[i].should_consider = false;
1974+ }
1975+ debug_matches.num_model_matches = 0;
1976+
1977+ if (deshake_ctx->debug_on) {
1978+ av_log(avctx, AV_LOG_VERBOSE,
1979+ "\n[ALERT] No motion data found in queue_frame, motion reset to 0\n\n"
1980+ );
1981+ }
1982+
1983+ goto end;
1984+
1985+end:
1986+ // Swap the descriptor buffers (we don't need the previous frame's descriptors
1987+ // again so we will use that space for the next frame's descriptors)
1988+ temp = deshake_ctx->prev_descriptors;
1989+ deshake_ctx->prev_descriptors = deshake_ctx->descriptors;
1990+ deshake_ctx->descriptors = temp;
1991+
1992+ // Same for the refined features
1993+ temp = deshake_ctx->prev_refined_features;
1994+ deshake_ctx->prev_refined_features = deshake_ctx->refined_features;
1995+ deshake_ctx->refined_features = temp;
1996+
1997+ if (deshake_ctx->debug_on) {
1998+ if (num_vectors == 0) {
1999+ debug_matches.matches = NULL;
2000+ } else {
2001+ debug_matches.matches = av_malloc_array(num_vectors, sizeof(MotionVector));
2002+
2003+ if (!debug_matches.matches) {
2004+ err = AVERROR(ENOMEM);
2005+ goto fail;
2006+ }
2007+ }
2008+
2009+ for (int i = 0; i < num_vectors; i++) {
2010+ debug_matches.matches[i] = deshake_ctx->matches_contig_host[i];
2011+ }
2012+ debug_matches.num_matches = num_vectors;
2013+
2014+ av_fifo_generic_write(
2015+ deshake_ctx->abs_motion.debug_matches,
2016+ &debug_matches,
2017+ sizeof(DebugMatches),
2018+ NULL
2019+ );
2020+ }
2021+
2022+ for (int i = 0; i < RingbufCount; i++) {
2023+ av_fifo_generic_write(
2024+ deshake_ctx->abs_motion.ringbuffers[i],
2025+ &new_vals[i],
2026+ sizeof(float),
2027+ NULL
2028+ );
2029+ }
2030+
2031+ return ff_framequeue_add(&deshake_ctx->fq, input_frame);
2032+
2033+fail:
2034+ clFinish(deshake_ctx->command_queue);
2035+ av_frame_free(&input_frame);
2036+ return err;
2037+}
2038+
2039+static int activate(AVFilterContext *ctx)
2040+{
2041+ AVFilterLink *inlink = ctx->inputs[0];
2042+ AVFilterLink *outlink = ctx->outputs[0];
2043+ DeshakeOpenCLContext *deshake_ctx = ctx->priv;
2044+ AVFrame *frame = NULL;
2045+ int ret, status;
2046+ int64_t pts;
2047+
2048+ FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink);
2049+
2050+ if (!deshake_ctx->eof) {
2051+ ret = ff_inlink_consume_frame(inlink, &frame);
2052+ if (ret < 0)
2053+ return ret;
2054+ if (ret > 0) {
2055+ if (!frame->hw_frames_ctx)
2056+ return AVERROR(EINVAL);
2057+
2058+ if (!deshake_ctx->initialized) {
2059+ ret = deshake_opencl_init(ctx);
2060+ if (ret < 0)
2061+ return ret;
2062+ }
2063+
2064+ // If there is no more space in the ringbuffers, remove the oldest
2065+ // values to make room for the new ones
2066+ if (av_fifo_space(deshake_ctx->abs_motion.ringbuffers[RingbufX]) == 0) {
2067+ for (int i = 0; i < RingbufCount; i++) {
2068+ av_fifo_drain(deshake_ctx->abs_motion.ringbuffers[i], sizeof(float));
2069+ }
2070+ }
2071+ ret = queue_frame(inlink, frame);
2072+ if (ret < 0)
2073+ return ret;
2074+ if (ret >= 0) {
2075+ // See if we have enough buffered frames to process one
2076+ //
2077+ // "enough" is half the smooth window of queued frames into the future
2078+ if (ff_framequeue_queued_frames(&deshake_ctx->fq) >= deshake_ctx->smooth_window / 2) {
2079+ return filter_frame(inlink, ff_framequeue_take(&deshake_ctx->fq));
2080+ }
2081+ }
2082+ }
2083+ }
2084+
2085+ if (!deshake_ctx->eof && ff_inlink_acknowledge_status(inlink, &status, &pts)) {
2086+ if (status == AVERROR_EOF) {
2087+ deshake_ctx->eof = true;
2088+ }
2089+ }
2090+
2091+ if (deshake_ctx->eof) {
2092+ // Finish processing the rest of the frames in the queue.
2093+ while(ff_framequeue_queued_frames(&deshake_ctx->fq) != 0) {
2094+ for (int i = 0; i < RingbufCount; i++) {
2095+ av_fifo_drain(deshake_ctx->abs_motion.ringbuffers[i], sizeof(float));
2096+ }
2097+
2098+ ret = filter_frame(inlink, ff_framequeue_take(&deshake_ctx->fq));
2099+ if (ret < 0) {
2100+ return ret;
2101+ }
2102+ }
2103+
2104+ if (deshake_ctx->debug_on) {
2105+ av_log(ctx, AV_LOG_VERBOSE,
2106+ "Average kernel execution times:\n"
2107+ "\t grayscale: %0.3f ms\n"
2108+ "\t harris_response: %0.3f ms\n"
2109+ "\t refine_features: %0.3f ms\n"
2110+ "\tbrief_descriptors: %0.3f ms\n"
2111+ "\tmatch_descriptors: %0.3f ms\n"
2112+ "\t transform: %0.3f ms\n"
2113+ "\t crop_upscale: %0.3f ms\n"
2114+ "Average buffer read times:\n"
2115+ "\t features buf: %0.3f ms\n",
2116+ averaged_event_time_ms(deshake_ctx->grayscale_time, deshake_ctx->curr_frame),
2117+ averaged_event_time_ms(deshake_ctx->harris_response_time, deshake_ctx->curr_frame),
2118+ averaged_event_time_ms(deshake_ctx->refine_features_time, deshake_ctx->curr_frame),
2119+ averaged_event_time_ms(deshake_ctx->brief_descriptors_time, deshake_ctx->curr_frame),
2120+ averaged_event_time_ms(deshake_ctx->match_descriptors_time, deshake_ctx->curr_frame),
2121+ averaged_event_time_ms(deshake_ctx->transform_time, deshake_ctx->curr_frame),
2122+ averaged_event_time_ms(deshake_ctx->crop_upscale_time, deshake_ctx->curr_frame),
2123+ averaged_event_time_ms(deshake_ctx->read_buf_time, deshake_ctx->curr_frame)
2124+ );
2125+ }
2126+
2127+ ff_outlink_set_status(outlink, AVERROR_EOF, deshake_ctx->duration);
2128+ return 0;
2129+ }
2130+
2131+ if (!deshake_ctx->eof) {
2132+ FF_FILTER_FORWARD_WANTED(outlink, inlink);
2133+ }
2134+
2135+ return FFERROR_NOT_READY;
2136+}
2137+
2138+static const AVFilterPad deshake_opencl_inputs[] = {
2139+ {
2140+ .name = "default",
2141+ .type = AVMEDIA_TYPE_VIDEO,
2142+ .config_props = &ff_opencl_filter_config_input,
2143+ },
2144+ { NULL }
2145+};
2146+
2147+static const AVFilterPad deshake_opencl_outputs[] = {
2148+ {
2149+ .name = "default",
2150+ .type = AVMEDIA_TYPE_VIDEO,
2151+ .config_props = &ff_opencl_filter_config_output,
2152+ },
2153+ { NULL }
2154+};
2155+
2156+#define OFFSET(x) offsetof(DeshakeOpenCLContext, x)
2157+#define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM
2158+
2159+static const AVOption deshake_opencl_options[] = {
2160+ {
2161+ "tripod", "simulates a tripod by preventing any camera movement whatsoever "
2162+ "from the original frame",
2163+ OFFSET(tripod_mode), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, FLAGS
2164+ },
2165+ {
2166+ "debug", "turn on additional debugging information",
2167+ OFFSET(debug_on), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, FLAGS
2168+ },
2169+ {
2170+ "adaptive_crop", "attempt to subtly crop borders to reduce mirrored content",
2171+ OFFSET(should_crop), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, FLAGS
2172+ },
2173+ {
2174+ "refine_features", "refine feature point locations at a sub-pixel level",
2175+ OFFSET(refine_features), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, FLAGS
2176+ },
2177+ {
2178+ "smooth_strength", "smoothing strength (0 attempts to adaptively determine optimal strength)",
2179+ OFFSET(smooth_percent), AV_OPT_TYPE_FLOAT, {.dbl = 0.0f}, 0.0f, 1.0f, FLAGS
2180+ },
2181+ {
2182+ "smooth_window_multiplier", "multiplier for number of frames to buffer for motion data",
2183+ OFFSET(smooth_window_multiplier), AV_OPT_TYPE_FLOAT, {.dbl = 2.0}, 0.1, 10.0, FLAGS
2184+ },
2185+ { NULL }
2186+};
2187+
2188+AVFILTER_DEFINE_CLASS(deshake_opencl);
2189+
2190+AVFilter ff_vf_deshake_opencl = {
2191+ .name = "deshake_opencl",
2192+ .description = NULL_IF_CONFIG_SMALL("Feature-point based video stabilization filter"),
2193+ .priv_size = sizeof(DeshakeOpenCLContext),
2194+ .priv_class = &deshake_opencl_class,
2195+ .init = &ff_opencl_filter_init,
2196+ .uninit = &deshake_opencl_uninit,
2197+ .query_formats = &ff_opencl_filter_query_formats,
2198+ .activate = activate,
2199+ .inputs = deshake_opencl_inputs,
2200+ .outputs = deshake_opencl_outputs,
2201+ .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE
2202+};

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t