History

fate: add tests for psnr and ssim filter

Metadata filter output is passed through an Awk script comparing floats against reference values with specified "fuzz" tolerance to account for architectural differences (e.g. x86-32 vs. x86-64).

Tobias Rapp committed 9 years ago

Commit bee01ee1 parent f102a4e7 files · +237 −0Browse files

Unified · Split

7 files
  1. tests/fate-run.sh +9 −0
  2. tests/fate/filter-video.mak +14 −0
  3. tests/ref/fate/filter-refcmp-psnr-rgb +45 −0
  4. tests/ref/fate/filter-refcmp-psnr-yuv +45 −0
  5. tests/ref/fate/filter-refcmp-ssim-rgb +30 −0
  6. tests/ref/fate/filter-refcmp-ssim-yuv +30 −0
  7. tests/refcmp-metadata.awk +64 −0
tests/fate-run.sh+9−0
@@ -234,6 +234,15 @@ lavftest(){ · 233 unmodified lines
234234 ${base}/lavf-regression.sh $t lavf tests/vsynth1 "$target_exec" "$target_path" "$threads" "$thread_type" "$cpuflags" "$target_samples"
235235 }
236236
237+refcmp_metadata(){
238+ refcmp=$1
239+ pixfmt=$2
240+ fuzz=${3:-0.001}
241+ ffmpeg $FLAGS $ENC_OPTS \
242+ -lavfi "testsrc2=size=300x200:rate=1:duration=5,format=${pixfmt},split[ref][tmp];[tmp]avgblur=4[enc];[enc][ref]${refcmp},metadata=print:file=-" \
243+ -f null /dev/null | awk -v ref=${ref} -v fuzz=${fuzz} -f ${base}/refcmp-metadata.awk -
244+}
245+
237246 video_filter(){
238247 filters=$1
239248 shift
tests/fate/filter-video.mak+14−0
@@ -747,6 +747,20 @@ FATE_FILTER_SAMPLES-$(call ALLYES, MOV_DEMUXER H264_DECODER AAC_FIXED_DECODER PC · 746 unmodified lines
747747 fate-filter-meta-4560-rotate0: tests/data/file4560-override2rotate0.mov
748748 fate-filter-meta-4560-rotate0: CMD = framecrc -flags +bitexact -c:a aac_fixed -i $(TARGET_PATH)/tests/data/file4560-override2rotate0.mov
749749
750+REFCMP_DEPS = FFMPEG LAVFI_INDEV TESTSRC2_FILTER AVGBLUR_FILTER METADATA_FILTER
751+
752+FATE_FILTER_SAMPLES-$(call ALLYES, $(REFCMP_DEPS) PSNR_FILTER) += fate-filter-refcmp-psnr-rgb
753+fate-filter-refcmp-psnr-rgb: CMD = refcmp_metadata psnr rgb24
754+
755+FATE_FILTER_SAMPLES-$(call ALLYES, $(REFCMP_DEPS) PSNR_FILTER) += fate-filter-refcmp-psnr-yuv
756+fate-filter-refcmp-psnr-yuv: CMD = refcmp_metadata psnr yuv422p
757+
758+FATE_FILTER_SAMPLES-$(call ALLYES, $(REFCMP_DEPS) SSIM_FILTER) += fate-filter-refcmp-ssim-rgb
759+fate-filter-refcmp-ssim-rgb: CMD = refcmp_metadata ssim rgb24
760+
761+FATE_FILTER_SAMPLES-$(call ALLYES, $(REFCMP_DEPS) SSIM_FILTER) += fate-filter-refcmp-ssim-yuv
762+fate-filter-refcmp-ssim-yuv: CMD = refcmp_metadata ssim yuv422p
763+
750764 FATE_SAMPLES_FFPROBE += $(FATE_METADATA_FILTER-yes)
751765 FATE_SAMPLES_FFMPEG += $(FATE_FILTER_SAMPLES-yes)
752766 FATE_FFMPEG += $(FATE_FILTER-yes)
tests/ref/fate/filter-refcmp-psnr-rgb+45−0
@@ -0,0 +1,45 @@
1+frame:0 pts:0 pts_time:0
2+lavfi.psnr.mse.r=1381.80
3+lavfi.psnr.psnr.r=16.73
4+lavfi.psnr.mse.g=896.00
5+lavfi.psnr.psnr.g=18.61
6+lavfi.psnr.mse.b=277.38
7+lavfi.psnr.psnr.b=23.70
8+lavfi.psnr.mse_avg=851.73
9+lavfi.psnr.psnr_avg=18.83
10+frame:1 pts:1 pts_time:1
11+lavfi.psnr.mse.r=1380.37
12+lavfi.psnr.psnr.r=16.73
13+lavfi.psnr.mse.g=975.91
14+lavfi.psnr.psnr.g=18.24
15+lavfi.psnr.mse.b=435.72
16+lavfi.psnr.psnr.b=21.74
17+lavfi.psnr.mse_avg=930.67
18+lavfi.psnr.psnr_avg=18.44
19+frame:2 pts:2 pts_time:2
20+lavfi.psnr.mse.r=1403.20
21+lavfi.psnr.psnr.r=16.66
22+lavfi.psnr.mse.g=954.05
23+lavfi.psnr.psnr.g=18.34
24+lavfi.psnr.mse.b=494.22
25+lavfi.psnr.psnr.b=21.19
26+lavfi.psnr.mse_avg=950.49
27+lavfi.psnr.psnr_avg=18.35
28+frame:3 pts:3 pts_time:3
29+lavfi.psnr.mse.r=1452.80
30+lavfi.psnr.psnr.r=16.51
31+lavfi.psnr.mse.g=1001.02
32+lavfi.psnr.psnr.g=18.13
33+lavfi.psnr.mse.b=557.39
34+lavfi.psnr.psnr.b=20.67
35+lavfi.psnr.mse_avg=1003.74
36+lavfi.psnr.psnr_avg=18.11
37+frame:4 pts:4 pts_time:4
38+lavfi.psnr.mse.r=1401.25
39+lavfi.psnr.psnr.r=16.67
40+lavfi.psnr.mse.g=1009.80
41+lavfi.psnr.psnr.g=18.09
42+lavfi.psnr.mse.b=602.42
43+lavfi.psnr.psnr.b=20.33
44+lavfi.psnr.mse_avg=1004.49
45+lavfi.psnr.psnr_avg=18.11
tests/ref/fate/filter-refcmp-psnr-yuv+45−0
@@ -0,0 +1,45 @@
1+frame:0 pts:0 pts_time:0
2+lavfi.psnr.mse.y=222.06
3+lavfi.psnr.psnr.y=24.67
4+lavfi.psnr.mse.u=339.38
5+lavfi.psnr.psnr.u=22.82
6+lavfi.psnr.mse.v=705.41
7+lavfi.psnr.psnr.v=19.65
8+lavfi.psnr.mse_avg=372.23
9+lavfi.psnr.psnr_avg=22.42
10+frame:1 pts:1 pts_time:1
11+lavfi.psnr.mse.y=236.74
12+lavfi.psnr.psnr.y=24.39
13+lavfi.psnr.mse.u=416.17
14+lavfi.psnr.psnr.u=21.94
15+lavfi.psnr.mse.v=704.98
16+lavfi.psnr.psnr.v=19.65
17+lavfi.psnr.mse_avg=398.66
18+lavfi.psnr.psnr_avg=22.12
19+frame:2 pts:2 pts_time:2
20+lavfi.psnr.mse.y=234.79
21+lavfi.psnr.psnr.y=24.42
22+lavfi.psnr.mse.u=435.72
23+lavfi.psnr.psnr.u=21.74
24+lavfi.psnr.mse.v=699.60
25+lavfi.psnr.psnr.v=19.68
26+lavfi.psnr.mse_avg=401.23
27+lavfi.psnr.psnr_avg=22.10
28+frame:3 pts:3 pts_time:3
29+lavfi.psnr.mse.y=250.88
30+lavfi.psnr.psnr.y=24.14
31+lavfi.psnr.mse.u=479.73
32+lavfi.psnr.psnr.u=21.32
33+lavfi.psnr.mse.v=707.55
34+lavfi.psnr.psnr.v=19.63
35+lavfi.psnr.mse_avg=422.26
36+lavfi.psnr.psnr_avg=21.88
37+frame:4 pts:4 pts_time:4
38+lavfi.psnr.mse.y=241.05
39+lavfi.psnr.psnr.y=24.31
40+lavfi.psnr.mse.u=505.04
41+lavfi.psnr.psnr.u=21.10
42+lavfi.psnr.mse.v=716.00
43+lavfi.psnr.psnr.v=19.58
44+lavfi.psnr.mse_avg=425.79
45+lavfi.psnr.psnr_avg=21.84
tests/ref/fate/filter-refcmp-ssim-rgb+30−0
@@ -0,0 +1,30 @@
1+frame:0 pts:0 pts_time:0
2+lavfi.ssim.R=0.72
3+lavfi.ssim.G=0.76
4+lavfi.ssim.B=0.89
5+lavfi.ssim.All=0.79
6+lavfi.ssim.dB=6.74
7+frame:1 pts:1 pts_time:1
8+lavfi.ssim.R=0.70
9+lavfi.ssim.G=0.74
10+lavfi.ssim.B=0.85
11+lavfi.ssim.All=0.77
12+lavfi.ssim.dB=6.31
13+frame:2 pts:2 pts_time:2
14+lavfi.ssim.R=0.71
15+lavfi.ssim.G=0.75
16+lavfi.ssim.B=0.84
17+lavfi.ssim.All=0.76
18+lavfi.ssim.dB=6.29
19+frame:3 pts:3 pts_time:3
20+lavfi.ssim.R=0.70
21+lavfi.ssim.G=0.73
22+lavfi.ssim.B=0.83
23+lavfi.ssim.All=0.76
24+lavfi.ssim.dB=6.11
25+frame:4 pts:4 pts_time:4
26+lavfi.ssim.R=0.71
27+lavfi.ssim.G=0.74
28+lavfi.ssim.B=0.80
29+lavfi.ssim.All=0.75
30+lavfi.ssim.dB=6.05
tests/ref/fate/filter-refcmp-ssim-yuv+30−0
@@ -0,0 +1,30 @@
1+frame:0 pts:0 pts_time:0
2+lavfi.ssim.Y=0.80
3+lavfi.ssim.U=0.76
4+lavfi.ssim.V=0.69
5+lavfi.ssim.All=0.76
6+lavfi.ssim.dB=6.25
7+frame:1 pts:1 pts_time:1
8+lavfi.ssim.Y=0.80
9+lavfi.ssim.U=0.73
10+lavfi.ssim.V=0.68
11+lavfi.ssim.All=0.75
12+lavfi.ssim.dB=6.08
13+frame:2 pts:2 pts_time:2
14+lavfi.ssim.Y=0.80
15+lavfi.ssim.U=0.73
16+lavfi.ssim.V=0.68
17+lavfi.ssim.All=0.75
18+lavfi.ssim.dB=6.10
19+frame:3 pts:3 pts_time:3
20+lavfi.ssim.Y=0.79
21+lavfi.ssim.U=0.72
22+lavfi.ssim.V=0.68
23+lavfi.ssim.All=0.75
24+lavfi.ssim.dB=5.94
25+frame:4 pts:4 pts_time:4
26+lavfi.ssim.Y=0.80
27+lavfi.ssim.U=0.72
28+lavfi.ssim.V=0.68
29+lavfi.ssim.All=0.75
30+lavfi.ssim.dB=5.97
tests/refcmp-metadata.awk+64−0
@@ -0,0 +1,64 @@
1+# Compare metadata filter output containing float value strings to reference
2+# output data. Returns the whole reference data if delta of each value is below
3+# threshold, else returns the whole input data.
4+
5+function abs(val) {
6+ return ((val < 0.0) ? -val : val);
7+}
8+
9+function max(val1, val2) {
10+ return ((val1 >= val2) ? val1 : val2);
11+}
12+
13+function is_numeric_str(str) {
14+ return (str ~ /^[+-]?[0-9]*\.?[0-9]+$/);
15+}
16+
17+BEGIN {
18+ FS = "=";
19+ # check for "fuzz" (threshold) program parameter, else use default
20+ if (fuzz <= 0.0) {
21+ fuzz = 0.1;
22+ }
23+ # check for "ref" (reference file) program parameter
24+ if (ref) {
25+ ref_nr = 0;
26+ while ((getline line < ref) > 0) {
27+ ref_nr++;
28+ ref_lines[ref_nr] = line;
29+ if (split(line, fields) == 2 && is_numeric_str(fields[2])) {
30+ ref_keys[ref_nr] = fields[1];
31+ ref_vals[ref_nr] = fields[2] + 0; # convert to number
32+ }
33+ }
34+ close(ref);
35+ }
36+ delta_max = 0;
37+ result = (ref ? 1 : 0);
38+}
39+
40+{
41+ cmp_lines[NR] = $0;
42+ if (NF == 2 && is_numeric_str($2) && ref_vals[NR]) {
43+ val = $2 + 0; # convert to number
44+ delta = abs((val / ref_vals[NR]) - 1);
45+ delta_max = max(delta_max, delta);
46+ result = result && ($1 == ref_keys[NR]) && (delta <= fuzz);
47+ } else {
48+ result = result && ($0 == ref_lines[NR]);
49+ }
50+}
51+
52+END {
53+ if (result) {
54+ for (i = 1; i <= ref_nr; i++)
55+ print ref_lines[i];
56+ } else {
57+ for (i = 1; i <= NR; i++)
58+ print cmp_lines[i];
59+ if (NR != ref_nr)
60+ print "[refcmp] lines: " NR " != " ref_nr > "/dev/stderr";
61+ if (delta_max >= fuzz)
62+ print "[refcmp] delta_max: " delta_max " >= " fuzz > "/dev/stderr";
63+ }
64+}

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t