Help/UGens/MachineListening/MFCC.html

   1 <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
   2 <html>
   3 <head>
   4 <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
   5 <meta http-equiv="Content-Style-Type" content="text/css">
   6 <title></title>
   7 <meta name="Generator" content="Cocoa HTML Writer">
   8 <meta name="CocoaVersion" content="949.43">
   9 <style type="text/css">
  10 p.p1 {margin: 0.0px 0.0px 0.0px 0.0px; font: 18.0px Helvetica}
  11 p.p2 {margin: 0.0px 0.0px 0.0px 0.0px; font: 12.0px Helvetica; min-height: 14.0px}
  12 p.p3 {margin: 0.0px 0.0px 0.0px 0.0px; font: 12.0px Helvetica}
  13 p.p4 {margin: 0.0px 0.0px 0.0px 0.0px; font: 9.0px Monaco; color: #a41c14}
  14 p.p5 {margin: 0.0px 0.0px 0.0px 0.0px; font: 9.0px Monaco; min-height: 12.0px}
  15 p.p6 {margin: 0.0px 0.0px 0.0px 0.0px; font: 9.0px Monaco; color: #606060}
  16 p.p7 {margin: 0.0px 0.0px 0.0px 0.0px; font: 9.0px Monaco}
  17 p.p8 {margin: 0.0px 0.0px 0.0px 0.0px; font: 16.0px Times; min-height: 19.0px}
  18 span.s1 {color: #000000}
  19 span.s2 {color: #0018b8}
  20 span.s3 {color: #356f14}
  21 span.s4 {color: #606060}
  22 span.s5 {color: #a41c14}
  23 span.Apple-tab-span {white-space:pre}
  24 </style>
  25 </head>
  26 <body>
  27 <p class="p1"><b>MFCC<span class="Apple-tab-span">      </span><span class="Apple-tab-span">    </span><span class="Apple-tab-span">    </span><span class="Apple-tab-span">    </span>mel frequency cepstral coefficients</b></p>
  28 <p class="p2"><br></p>
  29 <p class="p3"><b>#coeff1, coeff2, ... = MFCC.kr(chain, numcoeff=13)</b></p>
  30 <p class="p2"><br></p>
  31 <p class="p3"><b>chain [fft] - </b>Audio input to track, which has been pre-analysed by the FFT UGen; see examples below for the expected FFT size</p>
  32 <p class="p3"><b>numcoeff [s] - </b>Number of coefficients, defaults to 13, maximum of 42; more efficient to use less of course! <span class="Apple-converted-space"> </span></p>
  33 <p class="p2"><br></p>
  34 <p class="p3">Generates a set of MFCCs; these are obtained from a band-based frequency representation (using the Mel scale by default), and then a discrete cosine transform (DCT). The DCT is an efficient approximation for principal components analysis, so that it allows a compression, or reduction of dimensionality, of the data, in this case reducing 42 band readings to a smaller set of MFCCs. A small number of features (the coefficients) end up describing the spectrum. The MFCCs are commonly used as timbral descriptors.<span class="Apple-converted-space"> </span></p>
  35 <p class="p2"><br></p>
  36 <p class="p3">Output values are somewhat normalised for the range 0.0 to 1.0, but there are no guarantees on exact conformance to this. Commonly, the first coefficient will be the highest value.<span class="Apple-converted-space"> </span></p>
  37 <p class="p2"><br></p>
  38 <p class="p4">//Technical note: The 0th coefficient is not generated as it consists of multiplying all bands by 1 and summing</p>
  39 <p class="p5"><br></p>
  40 <p class="p5"><br></p>
  41 <p class="p4">//assumes hop of half fftsize, fine</p>
  42 <p class="p4"><span class="s1">b = </span><span class="s2">Buffer</span><span class="s1">.alloc(s,1024,1); </span>//for sampling rates 44100 and 48000</p>
  43 <p class="p4">//b = Buffer.alloc(s,2048,1); //for sampling rates 88200 and 96000</p>
  44 <p class="p5"><br></p>
  45 <p class="p6"><span class="s1">d=</span><span class="s2">Buffer</span><span class="s1">.read(s,</span>"sounds/a11wlk01.wav"<span class="s1">);</span></p>
  46 <p class="p5"><br></p>
  47 <p class="p5"><br></p>
  48 <p class="p7">(</p>
  49 <p class="p7">x= {</p>
  50 <p class="p7"><span class="s2">var</span> in, fft, array;</p>
  51 <p class="p5"><br></p>
  52 <p class="p4">//in= PlayBuf.ar(1,d,BufRateScale.kr(d),1,0,1);</p>
  53 <p class="p5"><br></p>
  54 <p class="p7">in = <span class="s2">SoundIn</span>.ar(0);<span class="Apple-converted-space"> </span></p>
  55 <p class="p5"><br></p>
  56 <p class="p7">fft = <span class="s2">FFT</span>(b, in);</p>
  57 <p class="p5"><br></p>
  58 <p class="p7">array=<span class="s2">MFCC</span>.kr(fft);<span class="Apple-converted-space"> </span></p>
  59 <p class="p5"><br></p>
  60 <p class="p7">array.size.postln;<span class="Apple-converted-space"> </span></p>
  61 <p class="p5"><br></p>
  62 <p class="p7"><span class="s2">Out</span>.kr(0,array);<span class="Apple-converted-space"> </span></p>
  63 <p class="p5"><br></p>
  64 <p class="p7"><span class="s2">Out</span>.ar(0,<span class="s2">Pan2</span>.ar(in));<span class="Apple-converted-space"> </span></p>
  65 <p class="p7">}.play</p>
  66 <p class="p7">)</p>
  67 <p class="p5"><br></p>
  68 <p class="p5"><br></p>
  69 <p class="p7">c= <span class="s2">Bus</span>.new(<span class="s3">'control'</span>, 0, 13);<span class="Apple-converted-space"> </span></p>
  70 <p class="p5"><br></p>
  71 <p class="p4">//poll coefficients</p>
  72 <p class="p7">c.getn(13,{<span class="s2">arg</span> val; {val.plot;}.defer});<span class="Apple-converted-space"> </span></p>
  73 <p class="p5"><br></p>
  74 <p class="p5"><br></p>
  75 <p class="p5"><br></p>
  76 <p class="p4">//Continuous graphical display of MFCC values; free routine before closing window</p>
  77 <p class="p5"><br></p>
  78 <p class="p7">(</p>
  79 <p class="p7"><span class="s2">var</span> ms;<span class="Apple-converted-space"> </span></p>
  80 <p class="p5"><br></p>
  81 <p class="p7">w=<span class="s2">Window</span>.new(<span class="s4">"Thirteen MFCC coefficients"</span>, <span class="s2">Rect</span>(200,400,300,300));</p>
  82 <p class="p5"><br></p>
  83 <p class="p7">ms= <span class="s2">Mu</span>ltiSliderView.new(w, <span class="s2">Rect</span>(10,10,260,280));</p>
  84 <p class="p5"><br></p>
  85 <p class="p7">ms.value_(<span class="s2">Array</span>.fill(13,0.0));</p>
  86 <p class="p7">ms.valueThumbSize_(20.0);</p>
  87 <p class="p7">ms.indexThumbSize_(20.0);</p>
  88 <p class="p7">ms.gap_(0);</p>
  89 <p class="p5"><br></p>
  90 <p class="p7">w.front;</p>
  91 <p class="p5"><br></p>
  92 <p class="p7">r= {</p>
  93 <p class="p5"><br></p>
  94 <p class="p7"><span class="s2">inf</span>.do{</p>
  95 <p class="p5"><br></p>
  96 <p class="p7">c.getn(13,{<span class="s2">arg</span> val; {ms.value_(val*0.9)}.defer});<span class="Apple-converted-space"> </span></p>
  97 <p class="p5"><br></p>
  98 <p class="p4"><span class="s1">0.04.wait; </span>//25 frames per second</p>
  99 <p class="p7">};</p>
 100 <p class="p5"><br></p>
 101 <p class="p7">}.fork;</p>
 102 <p class="p5"><br></p>
 103 <p class="p7">)</p>
 104 <p class="p5"><br></p>
 105 <p class="p5"><br></p>
 106 <p class="p4">//tidy up</p>
 107 <p class="p7">(</p>
 108 <p class="p7">r.stop;</p>
 109 <p class="p7">b.free;</p>
 110 <p class="p7">c.free;</p>
 111 <p class="p7">x.free;</p>
 112 <p class="p7">w.close;</p>
 113 <p class="p7">)</p>
 114 <p class="p8"><br></p>
 115 <p class="p8"><br></p>
 116 <p class="p8"><br></p>
 117 <p class="p8"><br></p>
 118 <p class="p8"><br></p>
 119 <p class="p3">Research notes: Drafts of an MFCC UGen were prepared by both Dan Stowell and Nick Collins; their various ideas are combined here in a cross platform compatible UGen. Mel scale spacing with triangular crossfade overlap is used by default for the bands, approximately tracking the human critical band spacing and bandwidth. Variants such as BFCC (Bark) and EFCC (ERB) given similar results in practice; the Mel scale as used here is the standard as adapted from the speech recognition literature and now applied in music information retrieval. <span class="Apple-converted-space"> </span></p>
 120 <p class="p5"><br></p>
 121 <p class="p4">//Calculating Mel Scale Bands; allow up to 42 coefficients, so up to 42 bands</p>
 122 <p class="p4">//first part of this code adapted from Dan Stowell and Jamie Bullock Mel scale implementation</p>
 123 <p class="p4">//could later add Bark and ERB options, and possibility of buffer of data to be passed to the UGen for alternative freq warpings</p>
 124 <p class="p7">(</p>
 125 <p class="p7"><span class="s2">var</span> mel_freq_max, mel_freq_min,freq_bw_mel, freq_bands, freq_max, freq_min;<span class="Apple-converted-space"> </span></p>
 126 <p class="p7"><span class="s2">var</span> mel_peak, lin_peak, fft_peak;<span class="Apple-converted-space"> </span></p>
 127 <p class="p7"><span class="s2">var</span> freqperbin;<span class="Apple-converted-space"> </span></p>
 128 <p class="p7"><span class="s2">var</span> fftbinstart, fftbinend, fftbinmult, fftbincumul;<span class="Apple-converted-space"> </span></p>
 129 <p class="p7"><span class="s2">var</span> pos, tmp;<span class="Apple-converted-space"> </span></p>
 130 <p class="p7"><span class="s2">var</span> sr, fftsize, halffftsize;<span class="Apple-converted-space"> </span></p>
 131 <p class="p7"><span class="s2">var</span> whichbandscale,lintoscale,scaletolin; <span class="Apple-converted-space"> </span></p>
 132 <p class="p5"><br></p>
 133 <p class="p7">freq_max=18000;</p>
 134 <p class="p7">freq_min=80;</p>
 135 <p class="p7">sr=48000; <span class="s5">//44100;<span class="Apple-converted-space"> </span></span></p>
 136 <p class="p7">fftsize=1024;<span class="Apple-converted-space"> </span></p>
 137 <p class="p7">halffftsize= fftsize.div(2);</p>
 138 <p class="p7">freq_bands=42;<span class="Apple-converted-space"> </span></p>
 139 <p class="p5"><br></p>
 140 <p class="p4">//whichbandscale= 0 ;<span class="Apple-converted-space">  </span>// 0=mel; 1=bark (CB) 2=ERB<span class="Apple-converted-space"> </span></p>
 141 <p class="p4">//</p>
 142 <p class="p4">//lintoscale = {arg freq;<span class="Apple-converted-space"> </span></p>
 143 <p class="p4">//switch(whichbandscale,0,{1127 * log(1 + (freq / 700))},1,{},2,{}).value</p>
 144 <p class="p4">//};<span class="Apple-converted-space"> </span></p>
 145 <p class="p4">//scaletolin = {arg scalepos;<span class="Apple-converted-space"> </span></p>
 146 <p class="p4">//switch(whichbandscale,0,{700 * (exp(scalepos / 1127.0) -1);},1,{},2,{}).value</p>
 147 <p class="p4">//};</p>
 148 <p class="p5"><br></p>
 149 <p class="p7">lintoscale = {<span class="s2">arg</span> freq;<span class="Apple-converted-space"> </span></p>
 150 <p class="p7">1127 * log(1 + (freq / 700))</p>
 151 <p class="p7">};<span class="Apple-converted-space"> </span></p>
 152 <p class="p7">scaletolin = {<span class="s2">arg</span> scalepos;<span class="Apple-converted-space"> </span></p>
 153 <p class="p7">700 * (exp(scalepos / 1127.0) -1);</p>
 154 <p class="p7">};</p>
 155 <p class="p5"><br></p>
 156 <p class="p7">mel_freq_max = lintoscale.value(freq_max); <span class="s5">// 1127 * log(1 + (freq_max / 700));</span></p>
 157 <p class="p7">mel_freq_min = lintoscale.value(freq_min); <span class="s5">//1127 * log(1 + (freq_min / 700));</span></p>
 158 <p class="p7">freq_bw_mel = (mel_freq_max - mel_freq_min) / freq_bands;</p>
 159 <p class="p5"><br></p>
 160 <p class="p7">[mel_freq_max, mel_freq_min,freq_bw_mel ].postln;</p>
 161 <p class="p5"><br></p>
 162 <p class="p7">mel_peak= <span class="s2">Array</span>.fill(freq_bands+2,{0.0});<span class="Apple-converted-space"> </span></p>
 163 <p class="p7">lin_peak= <span class="s2">Array</span>.fill(freq_bands+2,{0.0});</p>
 164 <p class="p7">fft_peak= <span class="s2">Array</span>.fill(freq_bands+2,{0.0});</p>
 165 <p class="p5"><br></p>
 166 <p class="p7">freqperbin = sr/fftsize; <span class="s5">//SR/N</span></p>
 167 <p class="p5"><br></p>
 168 <p class="p7">mel_peak[0] = mel_freq_min;</p>
 169 <p class="p4"><span class="s1">lin_peak[0] = freq_min; </span>// === 700 * (exp(mel_peak[0] / 1127) - 1);</p>
 170 <p class="p7">fft_peak[0] = (lin_peak[0]/ freqperbin).asInteger;</p>
 171 <p class="p5"><br></p>
 172 <p class="p7">for(1, freq_bands + 1,{<span class="s2">|n|</span></p>
 173 <p class="p5"><br></p>
 174 <p class="p7"><span class="Apple-converted-space"> </span>mel_peak[n] = mel_peak[n - 1] + freq_bw_mel;</p>
 175 <p class="p7"><span class="Apple-converted-space"> </span>lin_peak[n] = scaletolin.value(mel_peak[n]); <span class="s5">//700 * (exp(mel_peak[n] / 1127.0) -1);</span></p>
 176 <p class="p7"><span class="Apple-converted-space"> </span>fft_peak[n] = ((lin_peak[n] / freqperbin).asInteger).min(halffftsize); <span class="s5">//fft size //rounds down here</span></p>
 177 <p class="p5"><br></p>
 178 <p class="p7">});<span class="Apple-converted-space"> </span></p>
 179 <p class="p5"><br></p>
 180 <p class="p4">//Post &lt;&lt; mel_peak &lt;&lt; nl;</p>
 181 <p class="p4">//Post &lt;&lt; lin_peak &lt;&lt; nl;</p>
 182 <p class="p4">//Post &lt;&lt; fft_peak &lt;&lt; nl;</p>
 183 <p class="p5"><br></p>
 184 <p class="p4">//<span class="Apple-converted-space">  </span>[2 / (lin_peak[freq_bands + 1] - lin_peak[freq_bands-1]), 1.0 / (2 / (lin_peak[2] - lin_peak[0]))].postln;</p>
 185 <p class="p5"><br></p>
 186 <p class="p7">fftbinstart = <span class="s2">Array</span>.fill(freq_bands, {0});</p>
 187 <p class="p7">fftbinend = <span class="s2">Array</span>.fill(freq_bands, {0});</p>
 188 <p class="p7">fftbincumul = <span class="s2">Array</span>.fill(freq_bands+1, {0});</p>
 189 <p class="p7">fftbinmult = [];</p>
 190 <p class="p5"><br></p>
 191 <p class="p7">pos=0;<span class="Apple-converted-space"> </span></p>
 192 <p class="p5"><br></p>
 193 <p class="p7">freq_bands.do {<span class="s2">|i|</span></p>
 194 <p class="p5"><br></p>
 195 <p class="p4"><span class="s1"><span class="Apple-tab-span">    </span></span>//var normmult=1.0; //preserve power, don't modify band power by area <span class="Apple-converted-space"> </span></p>
 196 <p class="p7"><span class="Apple-tab-span">     </span><span class="s2">var</span> startbin, endbin, numbins, averager;<span class="Apple-converted-space"> </span></p>
 197 <p class="p5"><span class="Apple-tab-span">     </span></p>
 198 <p class="p7"><span class="Apple-tab-span">     </span>if(i==0,{</p>
 199 <p class="p7"><span class="Apple-tab-span">     </span>startbin= 0;<span class="Apple-converted-space"> </span></p>
 200 <p class="p7"><span class="Apple-tab-span">     </span>endbin= fft_peak[i+1]-1;</p>
 201 <p class="p7"><span class="Apple-tab-span">     </span>},{</p>
 202 <p class="p7"><span class="Apple-tab-span">     </span>startbin= fft_peak[i-1]+1;</p>
 203 <p class="p7"><span class="Apple-tab-span">     </span>endbin= fft_peak[i+1]-1;</p>
 204 <p class="p7"><span class="Apple-tab-span">     </span>});<span class="Apple-converted-space"> </span></p>
 205 <p class="p5"><span class="Apple-tab-span">     </span></p>
 206 <p class="p7"><span class="Apple-tab-span">     </span>numbins= endbin-startbin+1;<span class="Apple-converted-space"> </span></p>
 207 <p class="p7"><span class="Apple-tab-span">     </span>averager= 1.0/numbins;<span class="Apple-converted-space"> </span></p>
 208 <p class="p5"><span class="Apple-tab-span">     </span></p>
 209 <p class="p4"><span class="s1"><span class="Apple-tab-span">    </span></span>//linear crossfade (intended in power) between consecutive band centres</p>
 210 <p class="p5"><span class="Apple-tab-span">     </span></p>
 211 <p class="p7"><span class="Apple-tab-span">     </span>tmp= fft_peak[i]-startbin;<span class="Apple-converted-space"> </span></p>
 212 <p class="p5"><span class="Apple-tab-span">     </span></p>
 213 <p class="p4"><span class="s1"><span class="Apple-tab-span">    </span></span>//could divide by averager but I'm not convinced by the perceptual necessity for this?<span class="Apple-converted-space"> </span></p>
 214 <p class="p4"><span class="s1"><span class="Apple-tab-span">    </span></span>//ie fftbinmult = fftbinmult++(Array.series(tmp+1, 1.0/(tmp+1), 1.0/(tmp+1))*averager);<span class="Apple-converted-space"> </span></p>
 215 <p class="p5"><span class="Apple-tab-span">     </span></p>
 216 <p class="p7"><span class="Apple-tab-span">     </span>fftbinmult = fftbinmult++(<span class="s2">Array</span>.series(tmp+1, 1.0/(tmp+1), 1.0/(tmp+1)));<span class="Apple-converted-space"> </span></p>
 217 <p class="p5"><span class="Apple-tab-span">     </span></p>
 218 <p class="p7"><span class="Apple-tab-span">     </span>tmp= endbin- (fft_peak[i]);<span class="Apple-converted-space"> </span></p>
 219 <p class="p5"><span class="Apple-tab-span">     </span></p>
 220 <p class="p7"><span class="Apple-tab-span">     </span>fftbinmult = fftbinmult++(<span class="s2">Array</span>.series(tmp, 1.0+((-1.0)/(tmp+1)), (-1.0)/(tmp+1)));<span class="Apple-converted-space"> </span></p>
 221 <p class="p5"><span class="Apple-tab-span">     </span></p>
 222 <p class="p7"><span class="Apple-tab-span">     </span>fftbinstart[i]=startbin;</p>
 223 <p class="p7"><span class="Apple-tab-span">     </span>fftbinend[i]=endbin;<span class="Apple-converted-space"> </span></p>
 224 <p class="p7"><span class="Apple-tab-span">     </span>fftbincumul[i]= pos;</p>
 225 <p class="p5"><span class="Apple-tab-span">     </span></p>
 226 <p class="p7"><span class="Apple-tab-span">     </span>pos= pos+(endbin-startbin+1);</p>
 227 <p class="p7">};</p>
 228 <p class="p5"><br></p>
 229 <p class="p7">fftbincumul[freq_bands]= pos-1; <span class="Apple-converted-space"> </span></p>
 230 <p class="p5"><br></p>
 231 <p class="p7"><span class="s2">Post</span> &lt;&lt; fftbinstart &lt;&lt; nl;</p>
 232 <p class="p7"><span class="s2">Post</span> &lt;&lt; fftbinend &lt;&lt; nl;</p>
 233 <p class="p7"><span class="s2">Post</span> &lt;&lt; fftbincumul &lt;&lt; nl;</p>
 234 <p class="p7"><span class="s2">Post</span> &lt;&lt; fftbinmult &lt;&lt; nl;</p>
 235 <p class="p5"><br></p>
 236 <p class="p7">)</p>
 237 <p class="p5"><br></p>
 238 <p class="p5"><br></p>
 239 <p class="p4">//future work: see http://www.ling.su.se/STAFF/hartmut/bark.htm</p>
 240 <p class="p5"><br></p>
 241 <p class="p4">//Barks</p>
 242 <p class="p7">a= (26.81 / (1 + (1960 / ((100,200..22000))))) - 0.53</p>
 243 <p class="p7">a.plot</p>
 244 <p class="p5"><br></p>
 245 <p class="p4">//ERBs (rough calculation, only really valid under 6000Hz, real scale goes up to 42 rather than 37 in 22000 Hz)<span class="Apple-converted-space"> </span></p>
 246 <p class="p7">a= <span class="s2">Array</span>.fill(220,{<span class="s2">|i|</span> <span class="s2">var</span> f; f= i*100; 11.17*log((f + 312) / (f + 14675)) + 43.0});<span class="Apple-converted-space"> </span></p>
 247 <p class="p7">a.plot</p>
 248 <p class="p5"><br></p>
 249 <p class="p5"><span class="Apple-converted-space">    </span></p>
 250 <p class="p4">//generating DCT coefficients</p>
 251 <p class="p4">//don't generate i=0 coefficient since it<span class="Apple-converted-space"> </span></p>
 252 <p class="p7">a= <span class="s2">Array</span>.fill(42, {<span class="s2">|i|</span> cos(pi/42.0*((0..41)+0.5)*(i+1))});<span class="Apple-converted-space"> </span></p>
 253 <p class="p7"><span class="s2">Post</span> &lt;&lt; a.flatten &lt;&lt; nl;<span class="Apple-converted-space"> </span></p>
 254 <p class="p5"><br></p>
 255 <p class="p5"><br></p>
 256 <p class="p5"><br></p>
 257 <p class="p5"><br></p>
 258 <p class="p5"><br></p>
 259 <p class="p5"><br></p>
 260 <p class="p5"><br></p>
 261 </body>
 262 </html>