Add an optional filter_line3 to the available optimisations. filter_line3 is equivalent to filter_line, memcpy, filter_line filter_line shares quite a number of loads and some calculations in common with its next iteration and testing shows that using aarch64 neon filter_line3s performance is 30% better than two filter_lines and a memcpy. Adds a test for vf_bwdif filter_line3 to checkasm Rounds job start lines down to a multiple of 4. This means that if filter_line3 exists then filter_line will not sometimes be called once at the end of a slice depending on thread count. The final slice may do up to 3 extra lines but filter_edge is faster than filter_line so it is unlikely to create any noticable thread load variation.
Unified · Split
| @@ -35,6 +35,9 @@ typedef struct BWDIFContext { · 34 unmodified lines | ||
| 35 | 35 | void (*filter_edge)(void *dst, void *prev, void *cur, void *next, |
| 36 | 36 | int w, int prefs, int mrefs, int prefs2, int mrefs2, |
| 37 | 37 | int parity, int clip_max, int spat); |
| 38 | + void (*filter_line3)(void *dst, int dstride, | |
| 39 | + const void *prev, const void *cur, const void *next, int prefs, | |
| 40 | + int w, int parity, int clip_max); | |
| 38 | 41 | } BWDIFContext; |
| 39 | 42 | |
| 40 | 43 | void ff_bwdif_init_filter_line(BWDIFContext *bwdif, int bit_depth); |
| @@ -53,4 +56,8 @@ void ff_bwdif_filter_line_c(void *dst1, void *prev1, void *cur1, void *next1, · 12 unmodified lines | ||
| 53 | 56 | int prefs3, int mrefs3, int prefs4, int mrefs4, |
| 54 | 57 | int parity, int clip_max); |
| 55 | 58 | |
| 59 | +void ff_bwdif_filter_line3_c(void * dst1, int d_stride, | |
| 60 | + const void * prev1, const void * cur1, const void * next1, int s_stride, | |
| 61 | + int w, int parity, int clip_max); | |
| 62 | + | |
| 56 | 63 | #endif /* AVFILTER_BWDIF_H */ |
| @@ -150,6 +150,31 @@ void ff_bwdif_filter_line_c(void *dst1, void *prev1, void *cur1, void *next1, · 149 unmodified lines | ||
| 150 | 150 | FILTER2() |
| 151 | 151 | } |
| 152 | 152 | |
| 153 | +#define NEXT_LINE()\ | |
| 154 | + dst += d_stride; \ | |
| 155 | + prev += prefs; \ | |
| 156 | + cur += prefs; \ | |
| 157 | + next += prefs; | |
| 158 | + | |
| 159 | +void ff_bwdif_filter_line3_c(void * dst1, int d_stride, | |
| 160 | + const void * prev1, const void * cur1, const void * next1, int s_stride, | |
| 161 | + int w, int parity, int clip_max) | |
| 162 | +{ | |
| 163 | + const int prefs = s_stride; | |
| 164 | + uint8_t * dst = dst1; | |
| 165 | + const uint8_t * prev = prev1; | |
| 166 | + const uint8_t * cur = cur1; | |
| 167 | + const uint8_t * next = next1; | |
| 168 | + | |
| 169 | + ff_bwdif_filter_line_c(dst, (void*)prev, (void*)cur, (void*)next, w, | |
| 170 | + prefs, -prefs, prefs * 2, - prefs * 2, prefs * 3, -prefs * 3, prefs * 4, -prefs * 4, parity, clip_max); | |
| 171 | + NEXT_LINE(); | |
| 172 | + memcpy(dst, cur, w); | |
| 173 | + NEXT_LINE(); | |
| 174 | + ff_bwdif_filter_line_c(dst, (void*)prev, (void*)cur, (void*)next, w, | |
| 175 | + prefs, -prefs, prefs * 2, - prefs * 2, prefs * 3, -prefs * 3, prefs * 4, -prefs * 4, parity, clip_max); | |
| 176 | +} | |
| 177 | + | |
| 153 | 178 | void ff_bwdif_filter_edge_c(void *dst1, void *prev1, void *cur1, void *next1, |
| 154 | 179 | int w, int prefs, int mrefs, int prefs2, int mrefs2, |
| 155 | 180 | int parity, int clip_max, int spat) |
| @@ -212,6 +237,13 @@ static void filter_edge_16bit(void *dst1, void *prev1, void *cur1, void *next1, · 56 unmodified lines | ||
| 212 | 237 | FILTER2() |
| 213 | 238 | } |
| 214 | 239 | |
| 240 | +// Round job start line down to multiple of 4 so that if filter_line3 exists | |
| 241 | +// and the frame is a multiple of 4 high then filter_line will never be called | |
| 242 | +static inline int job_start(const int jobnr, const int nb_jobs, const int h) | |
| 243 | +{ | |
| 244 | + return jobnr >= nb_jobs ? h : ((h * jobnr) / nb_jobs) & ~3; | |
| 245 | +} | |
| 246 | + | |
| 215 | 247 | static int filter_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs) |
| 216 | 248 | { |
| 217 | 249 | BWDIFContext *s = ctx->priv; |
| @@ -221,8 +253,8 @@ static int filter_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs) · 3 unmodified lines | ||
| 221 | 253 | int clip_max = (1 << (yadif->csp->comp[td->plane].depth)) - 1; |
| 222 | 254 | int df = (yadif->csp->comp[td->plane].depth + 7) / 8; |
| 223 | 255 | int refs = linesize / df; |
| 224 | − int slice_start = (td->h * jobnr ) / nb_jobs; | |
| 225 | − int slice_end = (td->h * (jobnr+1)) / nb_jobs; | |
| 256 | + int slice_start = job_start(jobnr, nb_jobs, td->h); | |
| 257 | + int slice_end = job_start(jobnr + 1, nb_jobs, td->h); | |
| 226 | 258 | int y; |
| 227 | 259 | |
| 228 | 260 | for (y = slice_start; y < slice_end; y++) { |
| @@ -244,6 +276,11 @@ static int filter_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs) · 15 unmodified lines | ||
| 244 | 276 | refs << 1, -(refs << 1), |
| 245 | 277 | td->parity ^ td->tff, clip_max, |
| 246 | 278 | (y < 2) || ((y + 3) > td->h) ? 0 : 1); |
| 279 | + } else if (s->filter_line3 && y + 2 < slice_end && y + 6 < td->h) { | |
| 280 | + s->filter_line3(dst, td->frame->linesize[td->plane], | |
| 281 | + prev, cur, next, linesize, td->w, | |
| 282 | + td->parity ^ td->tff, clip_max); | |
| 283 | + y += 2; | |
| 247 | 284 | } else { |
| 248 | 285 | s->filter_line(dst, prev, cur, next, td->w, |
| 249 | 286 | refs, -refs, refs << 1, -(refs << 1), |
| @@ -280,7 +317,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dstpic, · 30 unmodified lines | ||
| 280 | 317 | td.plane = i; |
| 281 | 318 | |
| 282 | 319 | ff_filter_execute(ctx, filter_slice, &td, NULL, |
| 283 | − FFMIN(h, ff_filter_get_nb_threads(ctx))); | |
| 320 | + FFMIN((h+3)/4, ff_filter_get_nb_threads(ctx))); | |
| 284 | 321 | } |
| 285 | 322 | if (yadif->current_field == YADIF_FIELD_END) { |
| 286 | 323 | yadif->current_field = YADIF_FIELD_NORMAL; |
| @@ -357,6 +394,7 @@ static int config_props(AVFilterLink *link) · 70 unmodified lines | ||
| 357 | 394 | |
| 358 | 395 | av_cold void ff_bwdif_init_filter_line(BWDIFContext *s, int bit_depth) |
| 359 | 396 | { |
| 397 | + s->filter_line3 = 0; | |
| 360 | 398 | if (bit_depth > 8) { |
| 361 | 399 | s->filter_intra = filter_intra_16bit; |
| 362 | 400 | s->filter_line = filter_line_c_16bit; |
| @@ -28,6 +28,10 @@ · 27 unmodified lines | ||
| 28 | 28 | for (size_t i = 0; i < count; i++) \ |
| 29 | 29 | buf0[i] = buf1[i] = rnd() & mask |
| 30 | 30 | |
| 31 | +#define randomize_overflow_check(buf0, buf1, mask, count) \ | |
| 32 | + for (size_t i = 0; i < count; i++) \ | |
| 33 | + buf0[i] = buf1[i] = (rnd() & 1) != 0 ? mask : 0; | |
| 34 | + | |
| 31 | 35 | #define BODY(type, depth) \ |
| 32 | 36 | do { \ |
| 33 | 37 | type prev0[9*WIDTH], prev1[9*WIDTH]; \ |
| @@ -83,6 +87,83 @@ void checkasm_check_vf_bwdif(void) · 49 unmodified lines | ||
| 83 | 87 | report("bwdif10"); |
| 84 | 88 | } |
| 85 | 89 | |
| 90 | + if (!ctx_8.filter_line3) | |
| 91 | + ctx_8.filter_line3 = ff_bwdif_filter_line3_c; | |
| 92 | + | |
| 93 | + { | |
| 94 | + LOCAL_ALIGNED_16(uint8_t, prev0, [11*WIDTH]); | |
| 95 | + LOCAL_ALIGNED_16(uint8_t, prev1, [11*WIDTH]); | |
| 96 | + LOCAL_ALIGNED_16(uint8_t, next0, [11*WIDTH]); | |
| 97 | + LOCAL_ALIGNED_16(uint8_t, next1, [11*WIDTH]); | |
| 98 | + LOCAL_ALIGNED_16(uint8_t, cur0, [11*WIDTH]); | |
| 99 | + LOCAL_ALIGNED_16(uint8_t, cur1, [11*WIDTH]); | |
| 100 | + LOCAL_ALIGNED_16(uint8_t, dst0, [WIDTH*3]); | |
| 101 | + LOCAL_ALIGNED_16(uint8_t, dst1, [WIDTH*3]); | |
| 102 | + const int stride = WIDTH; | |
| 103 | + const int mask = (1<<8)-1; | |
| 104 | + int parity; | |
| 105 | + | |
| 106 | + for (parity = 0; parity != 2; ++parity) { | |
| 107 | + if (check_func(ctx_8.filter_line3, "bwdif8.line3.rnd.p%d", parity)) { | |
| 108 | + | |
| 109 | + declare_func(void, void * dst1, int d_stride, | |
| 110 | + const void * prev1, const void * cur1, const void * next1, int prefs, | |
| 111 | + int w, int parity, int clip_max); | |
| 112 | + | |
| 113 | + randomize_buffers(prev0, prev1, mask, 11*WIDTH); | |
| 114 | + randomize_buffers(next0, next1, mask, 11*WIDTH); | |
| 115 | + randomize_buffers( cur0, cur1, mask, 11*WIDTH); | |
| 116 | + | |
| 117 | + call_ref(dst0, stride, | |
| 118 | + prev0 + stride * 4, cur0 + stride * 4, next0 + stride * 4, stride, | |
| 119 | + WIDTH, parity, mask); | |
| 120 | + call_new(dst1, stride, | |
| 121 | + prev1 + stride * 4, cur1 + stride * 4, next1 + stride * 4, stride, | |
| 122 | + WIDTH, parity, mask); | |
| 123 | + | |
| 124 | + if (memcmp(dst0, dst1, WIDTH*3) | |
| 125 | + || memcmp(prev0, prev1, WIDTH*11) | |
| 126 | + || memcmp(next0, next1, WIDTH*11) | |
| 127 | + || memcmp( cur0, cur1, WIDTH*11)) | |
| 128 | + fail(); | |
| 129 | + | |
| 130 | + bench_new(dst1, stride, | |
| 131 | + prev1 + stride * 4, cur1 + stride * 4, next1 + stride * 4, stride, | |
| 132 | + WIDTH, parity, mask); | |
| 133 | + } | |
| 134 | + } | |
| 135 | + | |
| 136 | + // Use just 0s and ~0s to try to provoke bad cropping or overflow | |
| 137 | + // Parity makes no difference to this test so just test 0 | |
| 138 | + if (check_func(ctx_8.filter_line3, "bwdif8.line3.overflow")) { | |
| 139 | + | |
| 140 | + declare_func(void, void * dst1, int d_stride, | |
| 141 | + const void * prev1, const void * cur1, const void * next1, int prefs, | |
| 142 | + int w, int parity, int clip_max); | |
| 143 | + | |
| 144 | + randomize_overflow_check(prev0, prev1, mask, 11*WIDTH); | |
| 145 | + randomize_overflow_check(next0, next1, mask, 11*WIDTH); | |
| 146 | + randomize_overflow_check( cur0, cur1, mask, 11*WIDTH); | |
| 147 | + | |
| 148 | + call_ref(dst0, stride, | |
| 149 | + prev0 + stride * 4, cur0 + stride * 4, next0 + stride * 4, stride, | |
| 150 | + WIDTH, 0, mask); | |
| 151 | + call_new(dst1, stride, | |
| 152 | + prev1 + stride * 4, cur1 + stride * 4, next1 + stride * 4, stride, | |
| 153 | + WIDTH, 0, mask); | |
| 154 | + | |
| 155 | + if (memcmp(dst0, dst1, WIDTH*3) | |
| 156 | + || memcmp(prev0, prev1, WIDTH*11) | |
| 157 | + || memcmp(next0, next1, WIDTH*11) | |
| 158 | + || memcmp( cur0, cur1, WIDTH*11)) | |
| 159 | + fail(); | |
| 160 | + | |
| 161 | + // No point to benching | |
| 162 | + } | |
| 163 | + | |
| 164 | + report("bwdif8.line3"); | |
| 165 | + } | |
| 166 | + | |
| 86 | 167 | { |
| 87 | 168 | LOCAL_ALIGNED_16(uint8_t, prev0, [11*WIDTH]); |
| 88 | 169 | LOCAL_ALIGNED_16(uint8_t, prev1, [11*WIDTH]); |