History

python/tests/eval_pitch: add evaluation script against TONAS database

Paul Brossier committed 13 years ago

Commit e8bc8e91 parent 8c3f7171 file · +143 −0Browse files

Showing the changes to python (1 of 1 file). Show all

Unified · Split

python/tests/eval_pitch+143−0
@@ -0,0 +1,143 @@
1+#! /usr/bin/env python
2+
3+"""
4+Script to evaluate pitch algorithms against TONAS database.
5+
6+See http://mtg.upf.edu/download/datasets/tonas/
7+
8+Example run:
9+
10+ $ ./eval_pitch /path/to/TONAS/*/*.wav
11+ OK: 94.74% vx r: 96.87% vx f: 15.83% f0: 96.02% %12: 0.50% /path/to/TONAS/Deblas/01-D_AMairena.wav
12+ OK: 89.89% vx r: 93.21% vx f: 13.81% f0: 90.74% %12: 1.51% /path/to/TONAS/Deblas/02-D_ChanoLobato.wav
13+ OK: 96.02% vx r: 96.73% vx f: 10.91% f0: 96.42% %12: 0.00% /path/to/TONAS/Deblas/03-D_Chocolate.wav
14+ [...]
15+ OK: 82.35% vx r: 95.52% vx f: 67.09% f0: 89.80% %12: 0.95% /path/to/TONAS/Martinetes2/80-M2_Rancapinos.wav
16+ OK: 61.97% vx r: 85.71% vx f: 22.03% f0: 55.63% %12: 8.57% /path/to/TONAS/Martinetes2/81-M2_SDonday.wav
17+ OK: 75.26% vx r: 91.63% vx f: 27.27% f0: 75.99% %12: 5.05% /path/to/TONAS/Martinetes2/82-M2_TiaAnicalaPiriniaca.wav
18+ OK: 82.77% vx r: 92.74% vx f: 38.27% f0: 87.33% %12: 1.67% 69 files, total_length: 1177.69s, total runtime: 25.91s
19+
20+
21+"""
22+
23+import sys
24+import time
25+import os.path
26+import numpy
27+from utils import array_from_text_file, array_from_yaml_file
28+from aubio import source, pitch, freqtomidi
29+
30+start = time.time()
31+
32+freq_tol = .50 # more or less half a tone
33+
34+methods = ["default", "yinfft", "mcomb", "yin", "fcomb", "schmitt", "specacf"]
35+method = methods[0]
36+
37+downsample = 1
38+tolerance = 0.35
39+silence = -40.
40+skip = 1
41+if method in ["yinfft", "default"]:
42+ downsample = 1
43+ tolerance = 0.45
44+elif method == "mcomb":
45+ downsample = 4
46+elif method == "yin":
47+ downsample = 4
48+ tolerance = 0.2
49+
50+samplerate = 44100 / downsample
51+hop_s = 512 / downsample
52+win_s = 2048 / downsample
53+
54+def get_pitches (filename, samplerate = samplerate, win_s = win_s, hop_s = hop_s):
55+ s = source(filename, samplerate, hop_s)
56+ samplerate = s.samplerate
57+
58+ p = pitch(method, win_s, hop_s, samplerate)
59+ p.set_unit("freq")
60+ p.set_tolerance(tolerance)
61+ p.set_silence(silence)
62+
63+ # list of pitches, in samples
64+ pitches = []
65+
66+ # total number of frames read
67+ total_frames = 0
68+ while True:
69+ samples, read = s()
70+ new_pitch = p(samples)[0]
71+ pitches.append([total_frames/float(samplerate), new_pitch])
72+ total_frames += read
73+ if read < hop_s: break
74+ return numpy.array(pitches)
75+
76+total_correct_f0, total_correct_sil, total_missed, total_incorrect, total_fp, total_total = 0, 0, 0, 0, 0, 0
77+total_correct_chroma, total_voiced = 0, 0
78+for source_file in sys.argv[1:]:
79+ ground_truth_file = source_file.replace('.wav', '.f0.Corrected')
80+ if os.path.isfile(ground_truth_file):
81+ ground_truth = array_from_text_file(ground_truth_file)[:,[0,2]]
82+ experiment = get_pitches(source_file)
83+ # check that we have the same length, more or less one frame
84+ assert abs(len(ground_truth) - len(experiment)) < 2
85+ # align experiment by skipping first results
86+ experiment = experiment[skip:]
87+ experiment[:,0] -= experiment[0,0]
88+ # trim to shortest list
89+ maxlen = min(len(ground_truth), len(experiment))
90+ experiment = experiment[:maxlen]
91+ ground_truth = ground_truth[:maxlen]
92+ # get difference matrix
93+ diffmat = abs(experiment - ground_truth)
94+ # make sure we got the timing right
95+ assert max(diffmat[:,0]) < 10e-4, source_file
96+ truth_pitches = freqtomidi(ground_truth[:,1])
97+ exper_pitches = freqtomidi(experiment[:,1])
98+
99+ total = len(truth_pitches)
100+ unvoiced = len(truth_pitches[truth_pitches == 0])
101+ voiced = total - unvoiced
102+ correct_sil, fp, missed, correct_f0, correct_chroma, incorrect = 0, 0, 0, 0, 0, 0
103+ for a, b in zip(truth_pitches, exper_pitches):
104+ if a == 0 and b == 0:
105+ correct_sil += 1
106+ elif a == 0 and b != 0:
107+ fp += 1
108+ elif a != 0 and b == 0:
109+ missed += 1
110+ elif abs(b - a) < freq_tol:
111+ correct_f0 += 1
112+ elif abs(b - a) % 12. < freq_tol:
113+ correct_chroma += 1
114+ else:
115+ incorrect += 1
116+ assert correct_sil + fp + missed + correct_f0 + correct_chroma + incorrect == total
117+ assert unvoiced == correct_sil + fp
118+ assert voiced == missed + correct_f0 + correct_chroma + incorrect
119+ print "OK: %6s%%" % ("%.2f" % (100. * (correct_f0 + correct_sil) / total )),
120+ print "vx r: %6s%%" % ("%.2f" % (100. - 100. * missed / voiced)),
121+ print "vx f: %6s%%" % ("%.2f" % (100. * fp / unvoiced)),
122+ print "f0: %6s%%" % ("%.2f" % (100. * correct_f0 / voiced)),
123+ print "%%12: %6s%%" % ("%.2f" % (100. * correct_chroma / voiced)),
124+ print source_file
125+ total_correct_sil += correct_sil
126+ total_correct_f0 += correct_f0
127+ total_correct_chroma += correct_chroma
128+ total_missed += missed
129+ total_incorrect += incorrect
130+ total_fp += fp
131+ total_voiced += voiced
132+ total_total += total
133+ else:
134+ print "ERR", "could not find ground_truth_file", ground_truth_file
135+
136+print "OK: %6s%%" % ("%.2f" % (100. * (total_correct_f0 + total_correct_sil) / total_total )),
137+print "vx r: %6s%%" % ("%.2f" % (100. - 100. * total_missed / total_voiced)),
138+print "vx f: %6s%%" % ("%.2f" % (100. * (total_fp) / (total_correct_sil + total_fp))),
139+print "f0: %6s%%" % ("%.2f" % (100. * total_correct_f0 / total_voiced)),
140+print "%%12: %6s%%" % ("%.2f" % (100. * total_correct_chroma / total_voiced)),
141+print "%d files," % len(sys.argv[1:]),
142+print "total_length: %.2fs," % ((total_total * hop_s) / float(samplerate)),
143+print "total runtime: %.2fs" % (time.time() - start)

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t