History

fate: add tests for psnr and ssim filter

Metadata filter output is passed through an Awk script comparing floats against reference values with specified "fuzz" tolerance to account for architectural differences (e.g. x86-32 vs. x86-64).

Tobias Rapp committed 9 years ago

Commit bee01ee1 parent f102a4e7 files · +237 −0Browse files

Unified · Split

7 files
  1. tests/fate-run.sh +9 −0
  2. tests/fate/filter-video.mak +14 −0
  3. tests/ref/fate/filter-refcmp-psnr-rgb +45 −0
  4. tests/ref/fate/filter-refcmp-psnr-yuv +45 −0
  5. tests/ref/fate/filter-refcmp-ssim-rgb +30 −0
  6. tests/ref/fate/filter-refcmp-ssim-yuv +30 −0
  7. tests/refcmp-metadata.awk +64 −0
tests/fate-run.sh+9−0
@@ -234,6 +234,15 @@ lavftest(){ · 233 unmodified lines
234 ${base}/lavf-regression.sh $t lavf tests/vsynth1 "$target_exec" "$target_path" "$threads" "$thread_type" "$cpuflags" "$target_samples"234 ${base}/lavf-regression.sh $t lavf tests/vsynth1 "$target_exec" "$target_path" "$threads" "$thread_type" "$cpuflags" "$target_samples"
235}235}
236 236
237refcmp_metadata(){
238 refcmp=$1
239 pixfmt=$2
240 fuzz=${3:-0.001}
241 ffmpeg $FLAGS $ENC_OPTS \
242 -lavfi "testsrc2=size=300x200:rate=1:duration=5,format=${pixfmt},split[ref][tmp];[tmp]avgblur=4[enc];[enc][ref]${refcmp},metadata=print:file=-" \
243 -f null /dev/null | awk -v ref=${ref} -v fuzz=${fuzz} -f ${base}/refcmp-metadata.awk -
244}
245
237video_filter(){246video_filter(){
238 filters=$1247 filters=$1
239 shift248 shift
tests/fate/filter-video.mak+14−0
@@ -747,6 +747,20 @@ FATE_FILTER_SAMPLES-$(call ALLYES, MOV_DEMUXER H264_DECODER AAC_FIXED_DECODER PC · 746 unmodified lines
747fate-filter-meta-4560-rotate0: tests/data/file4560-override2rotate0.mov747fate-filter-meta-4560-rotate0: tests/data/file4560-override2rotate0.mov
748fate-filter-meta-4560-rotate0: CMD = framecrc -flags +bitexact -c:a aac_fixed -i $(TARGET_PATH)/tests/data/file4560-override2rotate0.mov748fate-filter-meta-4560-rotate0: CMD = framecrc -flags +bitexact -c:a aac_fixed -i $(TARGET_PATH)/tests/data/file4560-override2rotate0.mov
749 749
750REFCMP_DEPS = FFMPEG LAVFI_INDEV TESTSRC2_FILTER AVGBLUR_FILTER METADATA_FILTER
751
752FATE_FILTER_SAMPLES-$(call ALLYES, $(REFCMP_DEPS) PSNR_FILTER) += fate-filter-refcmp-psnr-rgb
753fate-filter-refcmp-psnr-rgb: CMD = refcmp_metadata psnr rgb24
754
755FATE_FILTER_SAMPLES-$(call ALLYES, $(REFCMP_DEPS) PSNR_FILTER) += fate-filter-refcmp-psnr-yuv
756fate-filter-refcmp-psnr-yuv: CMD = refcmp_metadata psnr yuv422p
757
758FATE_FILTER_SAMPLES-$(call ALLYES, $(REFCMP_DEPS) SSIM_FILTER) += fate-filter-refcmp-ssim-rgb
759fate-filter-refcmp-ssim-rgb: CMD = refcmp_metadata ssim rgb24
760
761FATE_FILTER_SAMPLES-$(call ALLYES, $(REFCMP_DEPS) SSIM_FILTER) += fate-filter-refcmp-ssim-yuv
762fate-filter-refcmp-ssim-yuv: CMD = refcmp_metadata ssim yuv422p
763
750FATE_SAMPLES_FFPROBE += $(FATE_METADATA_FILTER-yes)764FATE_SAMPLES_FFPROBE += $(FATE_METADATA_FILTER-yes)
751FATE_SAMPLES_FFMPEG += $(FATE_FILTER_SAMPLES-yes)765FATE_SAMPLES_FFMPEG += $(FATE_FILTER_SAMPLES-yes)
752FATE_FFMPEG += $(FATE_FILTER-yes)766FATE_FFMPEG += $(FATE_FILTER-yes)
tests/ref/fate/filter-refcmp-psnr-rgb+45−0
@@ -0,0 +1,45 @@
1frame:0 pts:0 pts_time:0
2lavfi.psnr.mse.r=1381.80
3lavfi.psnr.psnr.r=16.73
4lavfi.psnr.mse.g=896.00
5lavfi.psnr.psnr.g=18.61
6lavfi.psnr.mse.b=277.38
7lavfi.psnr.psnr.b=23.70
8lavfi.psnr.mse_avg=851.73
9lavfi.psnr.psnr_avg=18.83
10frame:1 pts:1 pts_time:1
11lavfi.psnr.mse.r=1380.37
12lavfi.psnr.psnr.r=16.73
13lavfi.psnr.mse.g=975.91
14lavfi.psnr.psnr.g=18.24
15lavfi.psnr.mse.b=435.72
16lavfi.psnr.psnr.b=21.74
17lavfi.psnr.mse_avg=930.67
18lavfi.psnr.psnr_avg=18.44
19frame:2 pts:2 pts_time:2
20lavfi.psnr.mse.r=1403.20
21lavfi.psnr.psnr.r=16.66
22lavfi.psnr.mse.g=954.05
23lavfi.psnr.psnr.g=18.34
24lavfi.psnr.mse.b=494.22
25lavfi.psnr.psnr.b=21.19
26lavfi.psnr.mse_avg=950.49
27lavfi.psnr.psnr_avg=18.35
28frame:3 pts:3 pts_time:3
29lavfi.psnr.mse.r=1452.80
30lavfi.psnr.psnr.r=16.51
31lavfi.psnr.mse.g=1001.02
32lavfi.psnr.psnr.g=18.13
33lavfi.psnr.mse.b=557.39
34lavfi.psnr.psnr.b=20.67
35lavfi.psnr.mse_avg=1003.74
36lavfi.psnr.psnr_avg=18.11
37frame:4 pts:4 pts_time:4
38lavfi.psnr.mse.r=1401.25
39lavfi.psnr.psnr.r=16.67
40lavfi.psnr.mse.g=1009.80
41lavfi.psnr.psnr.g=18.09
42lavfi.psnr.mse.b=602.42
43lavfi.psnr.psnr.b=20.33
44lavfi.psnr.mse_avg=1004.49
45lavfi.psnr.psnr_avg=18.11
tests/ref/fate/filter-refcmp-psnr-yuv+45−0
@@ -0,0 +1,45 @@
1frame:0 pts:0 pts_time:0
2lavfi.psnr.mse.y=222.06
3lavfi.psnr.psnr.y=24.67
4lavfi.psnr.mse.u=339.38
5lavfi.psnr.psnr.u=22.82
6lavfi.psnr.mse.v=705.41
7lavfi.psnr.psnr.v=19.65
8lavfi.psnr.mse_avg=372.23
9lavfi.psnr.psnr_avg=22.42
10frame:1 pts:1 pts_time:1
11lavfi.psnr.mse.y=236.74
12lavfi.psnr.psnr.y=24.39
13lavfi.psnr.mse.u=416.17
14lavfi.psnr.psnr.u=21.94
15lavfi.psnr.mse.v=704.98
16lavfi.psnr.psnr.v=19.65
17lavfi.psnr.mse_avg=398.66
18lavfi.psnr.psnr_avg=22.12
19frame:2 pts:2 pts_time:2
20lavfi.psnr.mse.y=234.79
21lavfi.psnr.psnr.y=24.42
22lavfi.psnr.mse.u=435.72
23lavfi.psnr.psnr.u=21.74
24lavfi.psnr.mse.v=699.60
25lavfi.psnr.psnr.v=19.68
26lavfi.psnr.mse_avg=401.23
27lavfi.psnr.psnr_avg=22.10
28frame:3 pts:3 pts_time:3
29lavfi.psnr.mse.y=250.88
30lavfi.psnr.psnr.y=24.14
31lavfi.psnr.mse.u=479.73
32lavfi.psnr.psnr.u=21.32
33lavfi.psnr.mse.v=707.55
34lavfi.psnr.psnr.v=19.63
35lavfi.psnr.mse_avg=422.26
36lavfi.psnr.psnr_avg=21.88
37frame:4 pts:4 pts_time:4
38lavfi.psnr.mse.y=241.05
39lavfi.psnr.psnr.y=24.31
40lavfi.psnr.mse.u=505.04
41lavfi.psnr.psnr.u=21.10
42lavfi.psnr.mse.v=716.00
43lavfi.psnr.psnr.v=19.58
44lavfi.psnr.mse_avg=425.79
45lavfi.psnr.psnr_avg=21.84
tests/ref/fate/filter-refcmp-ssim-rgb+30−0
@@ -0,0 +1,30 @@
1frame:0 pts:0 pts_time:0
2lavfi.ssim.R=0.72
3lavfi.ssim.G=0.76
4lavfi.ssim.B=0.89
5lavfi.ssim.All=0.79
6lavfi.ssim.dB=6.74
7frame:1 pts:1 pts_time:1
8lavfi.ssim.R=0.70
9lavfi.ssim.G=0.74
10lavfi.ssim.B=0.85
11lavfi.ssim.All=0.77
12lavfi.ssim.dB=6.31
13frame:2 pts:2 pts_time:2
14lavfi.ssim.R=0.71
15lavfi.ssim.G=0.75
16lavfi.ssim.B=0.84
17lavfi.ssim.All=0.76
18lavfi.ssim.dB=6.29
19frame:3 pts:3 pts_time:3
20lavfi.ssim.R=0.70
21lavfi.ssim.G=0.73
22lavfi.ssim.B=0.83
23lavfi.ssim.All=0.76
24lavfi.ssim.dB=6.11
25frame:4 pts:4 pts_time:4
26lavfi.ssim.R=0.71
27lavfi.ssim.G=0.74
28lavfi.ssim.B=0.80
29lavfi.ssim.All=0.75
30lavfi.ssim.dB=6.05
tests/ref/fate/filter-refcmp-ssim-yuv+30−0
@@ -0,0 +1,30 @@
1frame:0 pts:0 pts_time:0
2lavfi.ssim.Y=0.80
3lavfi.ssim.U=0.76
4lavfi.ssim.V=0.69
5lavfi.ssim.All=0.76
6lavfi.ssim.dB=6.25
7frame:1 pts:1 pts_time:1
8lavfi.ssim.Y=0.80
9lavfi.ssim.U=0.73
10lavfi.ssim.V=0.68
11lavfi.ssim.All=0.75
12lavfi.ssim.dB=6.08
13frame:2 pts:2 pts_time:2
14lavfi.ssim.Y=0.80
15lavfi.ssim.U=0.73
16lavfi.ssim.V=0.68
17lavfi.ssim.All=0.75
18lavfi.ssim.dB=6.10
19frame:3 pts:3 pts_time:3
20lavfi.ssim.Y=0.79
21lavfi.ssim.U=0.72
22lavfi.ssim.V=0.68
23lavfi.ssim.All=0.75
24lavfi.ssim.dB=5.94
25frame:4 pts:4 pts_time:4
26lavfi.ssim.Y=0.80
27lavfi.ssim.U=0.72
28lavfi.ssim.V=0.68
29lavfi.ssim.All=0.75
30lavfi.ssim.dB=5.97
tests/refcmp-metadata.awk+64−0
@@ -0,0 +1,64 @@
1# Compare metadata filter output containing float value strings to reference
2# output data. Returns the whole reference data if delta of each value is below
3# threshold, else returns the whole input data.
4
5function abs(val) {
6 return ((val < 0.0) ? -val : val);
7}
8
9function max(val1, val2) {
10 return ((val1 >= val2) ? val1 : val2);
11}
12
13function is_numeric_str(str) {
14 return (str ~ /^[+-]?[0-9]*\.?[0-9]+$/);
15}
16
17BEGIN {
18 FS = "=";
19 # check for "fuzz" (threshold) program parameter, else use default
20 if (fuzz <= 0.0) {
21 fuzz = 0.1;
22 }
23 # check for "ref" (reference file) program parameter
24 if (ref) {
25 ref_nr = 0;
26 while ((getline line < ref) > 0) {
27 ref_nr++;
28 ref_lines[ref_nr] = line;
29 if (split(line, fields) == 2 && is_numeric_str(fields[2])) {
30 ref_keys[ref_nr] = fields[1];
31 ref_vals[ref_nr] = fields[2] + 0; # convert to number
32 }
33 }
34 close(ref);
35 }
36 delta_max = 0;
37 result = (ref ? 1 : 0);
38}
39
40{
41 cmp_lines[NR] = $0;
42 if (NF == 2 && is_numeric_str($2) && ref_vals[NR]) {
43 val = $2 + 0; # convert to number
44 delta = abs((val / ref_vals[NR]) - 1);
45 delta_max = max(delta_max, delta);
46 result = result && ($1 == ref_keys[NR]) && (delta <= fuzz);
47 } else {
48 result = result && ($0 == ref_lines[NR]);
49 }
50}
51
52END {
53 if (result) {
54 for (i = 1; i <= ref_nr; i++)
55 print ref_lines[i];
56 } else {
57 for (i = 1; i <= NR; i++)
58 print cmp_lines[i];
59 if (NR != ref_nr)
60 print "[refcmp] lines: " NR " != " ref_nr > "/dev/stderr";
61 if (delta_max >= fuzz)
62 print "[refcmp] delta_max: " delta_max " >= " fuzz > "/dev/stderr";
63 }
64}

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t