Add an optional filter_line3 to the available optimisations. filter_line3 is equivalent to filter_line, memcpy, filter_line filter_line shares quite a number of loads and some calculations in common with its next iteration and testing shows that using aarch64 neon filter_line3s performance is 30% better than two filter_lines and a memcpy. Adds a test for vf_bwdif filter_line3 to checkasm Rounds job start lines down to a multiple of 4. This means that if filter_line3 exists then filter_line will not sometimes be called once at the end of a slice depending on thread count. The final slice may do up to 3 extra lines but filter_edge is faster than filter_line so it is unlikely to create any noticable thread load variation.
Unified · Split
| @@ -35,6 +35,9 @@ typedef struct BWDIFContext { · 34 unmodified lines | |||
| 35 | void (*filter_edge)(void *dst, void *prev, void *cur, void *next, | 35 | void (*filter_edge)(void *dst, void *prev, void *cur, void *next, |
| 36 | int w, int prefs, int mrefs, int prefs2, int mrefs2, | 36 | int w, int prefs, int mrefs, int prefs2, int mrefs2, |
| 37 | int parity, int clip_max, int spat); | 37 | int parity, int clip_max, int spat); |
| 38 | void (*filter_line3)(void *dst, int dstride, | ||
| 39 | const void *prev, const void *cur, const void *next, int prefs, | ||
| 40 | int w, int parity, int clip_max); | ||
| 38 | } BWDIFContext; | 41 | } BWDIFContext; |
| 39 | 42 | ||
| 40 | void ff_bwdif_init_filter_line(BWDIFContext *bwdif, int bit_depth); | 43 | void ff_bwdif_init_filter_line(BWDIFContext *bwdif, int bit_depth); |
| @@ -53,4 +56,8 @@ void ff_bwdif_filter_line_c(void *dst1, void *prev1, void *cur1, void *next1, · 12 unmodified lines | |||
| 53 | int prefs3, int mrefs3, int prefs4, int mrefs4, | 56 | int prefs3, int mrefs3, int prefs4, int mrefs4, |
| 54 | int parity, int clip_max); | 57 | int parity, int clip_max); |
| 55 | 58 | ||
| 59 | void ff_bwdif_filter_line3_c(void * dst1, int d_stride, | ||
| 60 | const void * prev1, const void * cur1, const void * next1, int s_stride, | ||
| 61 | int w, int parity, int clip_max); | ||
| 62 | |||
| 56 | #endif /* AVFILTER_BWDIF_H */ | 63 | #endif /* AVFILTER_BWDIF_H */ |
| @@ -150,6 +150,31 @@ void ff_bwdif_filter_line_c(void *dst1, void *prev1, void *cur1, void *next1, · 149 unmodified lines | |||
| 150 | FILTER2() | 150 | FILTER2() |
| 151 | } | 151 | } |
| 152 | 152 | ||
| 153 | #define NEXT_LINE()\ | ||
| 154 | dst += d_stride; \ | ||
| 155 | prev += prefs; \ | ||
| 156 | cur += prefs; \ | ||
| 157 | next += prefs; | ||
| 158 | |||
| 159 | void ff_bwdif_filter_line3_c(void * dst1, int d_stride, | ||
| 160 | const void * prev1, const void * cur1, const void * next1, int s_stride, | ||
| 161 | int w, int parity, int clip_max) | ||
| 162 | { | ||
| 163 | const int prefs = s_stride; | ||
| 164 | uint8_t * dst = dst1; | ||
| 165 | const uint8_t * prev = prev1; | ||
| 166 | const uint8_t * cur = cur1; | ||
| 167 | const uint8_t * next = next1; | ||
| 168 | |||
| 169 | ff_bwdif_filter_line_c(dst, (void*)prev, (void*)cur, (void*)next, w, | ||
| 170 | prefs, -prefs, prefs * 2, - prefs * 2, prefs * 3, -prefs * 3, prefs * 4, -prefs * 4, parity, clip_max); | ||
| 171 | NEXT_LINE(); | ||
| 172 | memcpy(dst, cur, w); | ||
| 173 | NEXT_LINE(); | ||
| 174 | ff_bwdif_filter_line_c(dst, (void*)prev, (void*)cur, (void*)next, w, | ||
| 175 | prefs, -prefs, prefs * 2, - prefs * 2, prefs * 3, -prefs * 3, prefs * 4, -prefs * 4, parity, clip_max); | ||
| 176 | } | ||
| 177 | |||
| 153 | void ff_bwdif_filter_edge_c(void *dst1, void *prev1, void *cur1, void *next1, | 178 | void ff_bwdif_filter_edge_c(void *dst1, void *prev1, void *cur1, void *next1, |
| 154 | int w, int prefs, int mrefs, int prefs2, int mrefs2, | 179 | int w, int prefs, int mrefs, int prefs2, int mrefs2, |
| 155 | int parity, int clip_max, int spat) | 180 | int parity, int clip_max, int spat) |
| @@ -212,6 +237,13 @@ static void filter_edge_16bit(void *dst1, void *prev1, void *cur1, void *next1, · 56 unmodified lines | |||
| 212 | FILTER2() | 237 | FILTER2() |
| 213 | } | 238 | } |
| 214 | 239 | ||
| 240 | // Round job start line down to multiple of 4 so that if filter_line3 exists | ||
| 241 | // and the frame is a multiple of 4 high then filter_line will never be called | ||
| 242 | static inline int job_start(const int jobnr, const int nb_jobs, const int h) | ||
| 243 | { | ||
| 244 | return jobnr >= nb_jobs ? h : ((h * jobnr) / nb_jobs) & ~3; | ||
| 245 | } | ||
| 246 | |||
| 215 | static int filter_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs) | 247 | static int filter_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs) |
| 216 | { | 248 | { |
| 217 | BWDIFContext *s = ctx->priv; | 249 | BWDIFContext *s = ctx->priv; |
| @@ -221,8 +253,8 @@ static int filter_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs) · 3 unmodified lines | |||
| 221 | int clip_max = (1 << (yadif->csp->comp[td->plane].depth)) - 1; | 253 | int clip_max = (1 << (yadif->csp->comp[td->plane].depth)) - 1; |
| 222 | int df = (yadif->csp->comp[td->plane].depth + 7) / 8; | 254 | int df = (yadif->csp->comp[td->plane].depth + 7) / 8; |
| 223 | int refs = linesize / df; | 255 | int refs = linesize / df; |
| 224 | int slice_start = (td->h * jobnr ) / nb_jobs; | 256 | int slice_start = job_start(jobnr, nb_jobs, td->h); |
| 225 | int slice_end = (td->h * (jobnr+1)) / nb_jobs; | 257 | int slice_end = job_start(jobnr + 1, nb_jobs, td->h); |
| 226 | int y; | 258 | int y; |
| 227 | 259 | ||
| 228 | for (y = slice_start; y < slice_end; y++) { | 260 | for (y = slice_start; y < slice_end; y++) { |
| @@ -244,6 +276,11 @@ static int filter_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs) · 15 unmodified lines | |||
| 244 | refs << 1, -(refs << 1), | 276 | refs << 1, -(refs << 1), |
| 245 | td->parity ^ td->tff, clip_max, | 277 | td->parity ^ td->tff, clip_max, |
| 246 | (y < 2) || ((y + 3) > td->h) ? 0 : 1); | 278 | (y < 2) || ((y + 3) > td->h) ? 0 : 1); |
| 279 | } else if (s->filter_line3 && y + 2 < slice_end && y + 6 < td->h) { | ||
| 280 | s->filter_line3(dst, td->frame->linesize[td->plane], | ||
| 281 | prev, cur, next, linesize, td->w, | ||
| 282 | td->parity ^ td->tff, clip_max); | ||
| 283 | y += 2; | ||
| 247 | } else { | 284 | } else { |
| 248 | s->filter_line(dst, prev, cur, next, td->w, | 285 | s->filter_line(dst, prev, cur, next, td->w, |
| 249 | refs, -refs, refs << 1, -(refs << 1), | 286 | refs, -refs, refs << 1, -(refs << 1), |
| @@ -280,7 +317,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dstpic, · 30 unmodified lines | |||
| 280 | td.plane = i; | 317 | td.plane = i; |
| 281 | 318 | ||
| 282 | ff_filter_execute(ctx, filter_slice, &td, NULL, | 319 | ff_filter_execute(ctx, filter_slice, &td, NULL, |
| 283 | FFMIN(h, ff_filter_get_nb_threads(ctx))); | 320 | FFMIN((h+3)/4, ff_filter_get_nb_threads(ctx))); |
| 284 | } | 321 | } |
| 285 | if (yadif->current_field == YADIF_FIELD_END) { | 322 | if (yadif->current_field == YADIF_FIELD_END) { |
| 286 | yadif->current_field = YADIF_FIELD_NORMAL; | 323 | yadif->current_field = YADIF_FIELD_NORMAL; |
| @@ -357,6 +394,7 @@ static int config_props(AVFilterLink *link) · 70 unmodified lines | |||
| 357 | 394 | ||
| 358 | av_cold void ff_bwdif_init_filter_line(BWDIFContext *s, int bit_depth) | 395 | av_cold void ff_bwdif_init_filter_line(BWDIFContext *s, int bit_depth) |
| 359 | { | 396 | { |
| 397 | s->filter_line3 = 0; | ||
| 360 | if (bit_depth > 8) { | 398 | if (bit_depth > 8) { |
| 361 | s->filter_intra = filter_intra_16bit; | 399 | s->filter_intra = filter_intra_16bit; |
| 362 | s->filter_line = filter_line_c_16bit; | 400 | s->filter_line = filter_line_c_16bit; |
| @@ -28,6 +28,10 @@ · 27 unmodified lines | |||
| 28 | for (size_t i = 0; i < count; i++) \ | 28 | for (size_t i = 0; i < count; i++) \ |
| 29 | buf0[i] = buf1[i] = rnd() & mask | 29 | buf0[i] = buf1[i] = rnd() & mask |
| 30 | 30 | ||
| 31 | #define randomize_overflow_check(buf0, buf1, mask, count) \ | ||
| 32 | for (size_t i = 0; i < count; i++) \ | ||
| 33 | buf0[i] = buf1[i] = (rnd() & 1) != 0 ? mask : 0; | ||
| 34 | |||
| 31 | #define BODY(type, depth) \ | 35 | #define BODY(type, depth) \ |
| 32 | do { \ | 36 | do { \ |
| 33 | type prev0[9*WIDTH], prev1[9*WIDTH]; \ | 37 | type prev0[9*WIDTH], prev1[9*WIDTH]; \ |
| @@ -83,6 +87,83 @@ void checkasm_check_vf_bwdif(void) · 49 unmodified lines | |||
| 83 | report("bwdif10"); | 87 | report("bwdif10"); |
| 84 | } | 88 | } |
| 85 | 89 | ||
| 90 | if (!ctx_8.filter_line3) | ||
| 91 | ctx_8.filter_line3 = ff_bwdif_filter_line3_c; | ||
| 92 | |||
| 93 | { | ||
| 94 | LOCAL_ALIGNED_16(uint8_t, prev0, [11*WIDTH]); | ||
| 95 | LOCAL_ALIGNED_16(uint8_t, prev1, [11*WIDTH]); | ||
| 96 | LOCAL_ALIGNED_16(uint8_t, next0, [11*WIDTH]); | ||
| 97 | LOCAL_ALIGNED_16(uint8_t, next1, [11*WIDTH]); | ||
| 98 | LOCAL_ALIGNED_16(uint8_t, cur0, [11*WIDTH]); | ||
| 99 | LOCAL_ALIGNED_16(uint8_t, cur1, [11*WIDTH]); | ||
| 100 | LOCAL_ALIGNED_16(uint8_t, dst0, [WIDTH*3]); | ||
| 101 | LOCAL_ALIGNED_16(uint8_t, dst1, [WIDTH*3]); | ||
| 102 | const int stride = WIDTH; | ||
| 103 | const int mask = (1<<8)-1; | ||
| 104 | int parity; | ||
| 105 | |||
| 106 | for (parity = 0; parity != 2; ++parity) { | ||
| 107 | if (check_func(ctx_8.filter_line3, "bwdif8.line3.rnd.p%d", parity)) { | ||
| 108 | |||
| 109 | declare_func(void, void * dst1, int d_stride, | ||
| 110 | const void * prev1, const void * cur1, const void * next1, int prefs, | ||
| 111 | int w, int parity, int clip_max); | ||
| 112 | |||
| 113 | randomize_buffers(prev0, prev1, mask, 11*WIDTH); | ||
| 114 | randomize_buffers(next0, next1, mask, 11*WIDTH); | ||
| 115 | randomize_buffers( cur0, cur1, mask, 11*WIDTH); | ||
| 116 | |||
| 117 | call_ref(dst0, stride, | ||
| 118 | prev0 + stride * 4, cur0 + stride * 4, next0 + stride * 4, stride, | ||
| 119 | WIDTH, parity, mask); | ||
| 120 | call_new(dst1, stride, | ||
| 121 | prev1 + stride * 4, cur1 + stride * 4, next1 + stride * 4, stride, | ||
| 122 | WIDTH, parity, mask); | ||
| 123 | |||
| 124 | if (memcmp(dst0, dst1, WIDTH*3) | ||
| 125 | || memcmp(prev0, prev1, WIDTH*11) | ||
| 126 | || memcmp(next0, next1, WIDTH*11) | ||
| 127 | || memcmp( cur0, cur1, WIDTH*11)) | ||
| 128 | fail(); | ||
| 129 | |||
| 130 | bench_new(dst1, stride, | ||
| 131 | prev1 + stride * 4, cur1 + stride * 4, next1 + stride * 4, stride, | ||
| 132 | WIDTH, parity, mask); | ||
| 133 | } | ||
| 134 | } | ||
| 135 | |||
| 136 | // Use just 0s and ~0s to try to provoke bad cropping or overflow | ||
| 137 | // Parity makes no difference to this test so just test 0 | ||
| 138 | if (check_func(ctx_8.filter_line3, "bwdif8.line3.overflow")) { | ||
| 139 | |||
| 140 | declare_func(void, void * dst1, int d_stride, | ||
| 141 | const void * prev1, const void * cur1, const void * next1, int prefs, | ||
| 142 | int w, int parity, int clip_max); | ||
| 143 | |||
| 144 | randomize_overflow_check(prev0, prev1, mask, 11*WIDTH); | ||
| 145 | randomize_overflow_check(next0, next1, mask, 11*WIDTH); | ||
| 146 | randomize_overflow_check( cur0, cur1, mask, 11*WIDTH); | ||
| 147 | |||
| 148 | call_ref(dst0, stride, | ||
| 149 | prev0 + stride * 4, cur0 + stride * 4, next0 + stride * 4, stride, | ||
| 150 | WIDTH, 0, mask); | ||
| 151 | call_new(dst1, stride, | ||
| 152 | prev1 + stride * 4, cur1 + stride * 4, next1 + stride * 4, stride, | ||
| 153 | WIDTH, 0, mask); | ||
| 154 | |||
| 155 | if (memcmp(dst0, dst1, WIDTH*3) | ||
| 156 | || memcmp(prev0, prev1, WIDTH*11) | ||
| 157 | || memcmp(next0, next1, WIDTH*11) | ||
| 158 | || memcmp( cur0, cur1, WIDTH*11)) | ||
| 159 | fail(); | ||
| 160 | |||
| 161 | // No point to benching | ||
| 162 | } | ||
| 163 | |||
| 164 | report("bwdif8.line3"); | ||
| 165 | } | ||
| 166 | |||
| 86 | { | 167 | { |
| 87 | LOCAL_ALIGNED_16(uint8_t, prev0, [11*WIDTH]); | 168 | LOCAL_ALIGNED_16(uint8_t, prev0, [11*WIDTH]); |
| 88 | LOCAL_ALIGNED_16(uint8_t, prev1, [11*WIDTH]); | 169 | LOCAL_ALIGNED_16(uint8_t, prev1, [11*WIDTH]); |