1 /********************************************************************
3 * THIS FILE IS PART OF THE OggVorbis SOFTWARE CODEC SOURCE CODE. *
4 * USE, DISTRIBUTION AND REPRODUCTION OF THIS LIBRARY SOURCE IS *
5 * GOVERNED BY A BSD-STYLE SOURCE LICENSE INCLUDED WITH THIS SOURCE *
6 * IN 'COPYING'. PLEASE READ THESE TERMS BEFORE DISTRIBUTING. *
8 * THE OggVorbis SOURCE CODE IS (C) COPYRIGHT 1994-2001 *
9 * by the Xiph.Org Foundation http://www.xiph.org/ *
11 ********************************************************************
13 function: utility functions for loading .vqh and .vqd files
14 last mod: $Id: bookutil.c 16037 2009-05-26 21:10:58Z xiphmont $
16 ********************************************************************/
25 /* A few little utils for reading files */
26 /* read a line. Use global, persistent buffering */
27 static char *linebuffer
=NULL
;
28 static int lbufsize
=0;
29 char *get_line(FILE *in
){
31 if(feof(in
))return NULL
;
37 if(sofar
+1>=lbufsize
){
40 linebuffer
=_ogg_malloc(lbufsize
);
43 linebuffer
=_ogg_realloc(linebuffer
,lbufsize
);
50 if(sofar
==0)return(NULL
);
51 /* fallthrough correct */
53 linebuffer
[sofar
]='\0';
57 linebuffer
[sofar
++]=c
;
58 linebuffer
[sofar
]='\0';
64 if(linebuffer
[0]=='#'){
72 /* read the next numerical value from the given file */
73 static char *value_line_buff
=NULL
;
75 int get_line_value(FILE *in
,float *value
){
78 if(!value_line_buff
)return(-1);
80 *value
=strtod(value_line_buff
, &next
);
81 if(next
==value_line_buff
){
86 while(*value_line_buff
>44)value_line_buff
++;
87 if(*value_line_buff
==44)value_line_buff
++;
92 int get_next_value(FILE *in
,float *value
){
94 if(get_line_value(in
,value
)){
95 value_line_buff
=get_line(in
);
96 if(!value_line_buff
)return(-1);
103 int get_next_ivalue(FILE *in
,long *ivalue
){
105 int ret
=get_next_value(in
,&value
);
110 static float sequence_base
=0.f
;
111 static int v_sofar
=0;
112 void reset_next_value(void){
113 value_line_buff
=NULL
;
118 char *setup_line(FILE *in
){
120 value_line_buff
=get_line(in
);
121 return(value_line_buff
);
125 int get_vector(codebook
*b
,FILE *in
,int start
, int n
,float *a
){
127 const static_codebook
*c
=b
->c
;
131 if(v_sofar
==n
|| get_line_value(in
,a
)){
133 if(get_next_value(in
,a
))
135 for(i
=0;i
<start
;i
++){
137 get_line_value(in
,a
);
141 for(i
=1;i
<c
->dim
;i
++)
142 if(get_line_value(in
,a
+i
))
146 float temp
=a
[c
->dim
-1];
147 for(i
=0;i
<c
->dim
;i
++)a
[i
]-=sequence_base
;
148 if(c
->q_sequencep
)sequence_base
=temp
;
158 /* read lines fromt he beginning until we find one containing the
160 char *find_seek_to(FILE *in
,char *s
){
163 char *line
=get_line(in
);
173 /* this reads the format as written by vqbuild/latticebuild; innocent
174 (legal) tweaking of the file that would not affect its valid
175 header-ness will break this routine */
177 codebook
*codebook_load(char *filename
){
178 codebook
*b
=_ogg_calloc(1,sizeof(codebook
));
179 static_codebook
*c
=(static_codebook
*)(b
->c
=_ogg_calloc(1,sizeof(static_codebook
)));
180 encode_aux_nearestmatch
*a
=NULL
;
181 encode_aux_threshmatch
*t
=NULL
;
182 encode_aux_pigeonhole
*p
=NULL
;
184 FILE *in
=fopen(filename
,"r");
189 fprintf(stderr
,"Couldn't open codebook %s\n",filename
);
193 /* find the codebook struct */
194 find_seek_to(in
,"static const static_codebook ");
196 /* get the major important values */
198 if(sscanf(line
,"%ld, %ld,",
199 &(c
->dim
),&(c
->entries
))!=2){
200 fprintf(stderr
,"1: syntax in %s in line:\t %s",filename
,line
);
205 if(sscanf(line
,"%d, %ld, %ld, %d, %d,",
206 &(c
->maptype
),&(c
->q_min
),&(c
->q_delta
),&(c
->q_quant
),
207 &(c
->q_sequencep
))!=5){
208 fprintf(stderr
,"1: syntax in %s in line:\t %s",filename
,line
);
212 /* find the auxiliary encode struct[s] (if any) */
213 if(find_seek_to(in
,"static const encode_aux_nearestmatch _vq_aux")){
215 c
->nearest_tree
=a
=_ogg_calloc(1,sizeof(encode_aux_nearestmatch
));
221 if(sscanf(line
,"%ld, %ld",&(a
->aux
),&(a
->alloc
))!=2){
222 fprintf(stderr
,"2: syntax in %s in line:\t %s",filename
,line
);
227 find_seek_to(in
,"static const long _vq_ptr0");
229 a
->ptr0
=_ogg_malloc(sizeof(long)*a
->aux
);
230 for(i
=0;i
<a
->aux
;i
++)
231 if(get_next_ivalue(in
,a
->ptr0
+i
)){
232 fprintf(stderr
,"out of data while reading codebook %s\n",filename
);
237 find_seek_to(in
,"static const long _vq_ptr1");
239 a
->ptr1
=_ogg_malloc(sizeof(long)*a
->aux
);
240 for(i
=0;i
<a
->aux
;i
++)
241 if(get_next_ivalue(in
,a
->ptr1
+i
)){
242 fprintf(stderr
,"out of data while reading codebook %s\n",filename
);
248 find_seek_to(in
,"static const long _vq_p_");
250 a
->p
=_ogg_malloc(sizeof(long)*a
->aux
);
251 for(i
=0;i
<a
->aux
;i
++)
252 if(get_next_ivalue(in
,a
->p
+i
)){
253 fprintf(stderr
,"out of data while reading codebook %s\n",filename
);
258 find_seek_to(in
,"static const long _vq_q_");
260 a
->q
=_ogg_malloc(sizeof(long)*a
->aux
);
261 for(i
=0;i
<a
->aux
;i
++)
262 if(get_next_ivalue(in
,a
->q
+i
)){
263 fprintf(stderr
,"out of data while reading codebook %s\n",filename
);
268 if(find_seek_to(in
,"static const encode_aux_threshmatch _vq_aux")){
270 c
->thresh_tree
=t
=_ogg_calloc(1,sizeof(encode_aux_threshmatch
));
274 if(sscanf(line
,"%d",&(t
->quantvals
))!=1){
275 fprintf(stderr
,"3: syntax in %s in line:\t %s",filename
,line
);
279 if(sscanf(line
,"%d",&(t
->threshvals
))!=1){
280 fprintf(stderr
,"4: syntax in %s in line:\t %s",filename
,line
);
283 /* load quantthresh */
284 find_seek_to(in
,"static const float _vq_quantthresh_");
286 t
->quantthresh
=_ogg_malloc(sizeof(float)*t
->threshvals
);
287 for(i
=0;i
<t
->threshvals
-1;i
++)
288 if(get_next_value(in
,t
->quantthresh
+i
)){
289 fprintf(stderr
,"out of data 1 while reading codebook %s\n",filename
);
293 find_seek_to(in
,"static const long _vq_quantmap_");
295 t
->quantmap
=_ogg_malloc(sizeof(long)*t
->threshvals
);
296 for(i
=0;i
<t
->threshvals
;i
++)
297 if(get_next_ivalue(in
,t
->quantmap
+i
)){
298 fprintf(stderr
,"out of data 2 while reading codebook %s\n",filename
);
303 if(find_seek_to(in
,"static const encode_aux_pigeonhole _vq_aux")){
306 c
->pigeon_tree
=p
=_ogg_calloc(1,sizeof(encode_aux_pigeonhole
));
308 if(sscanf(line
,"%f, %f, %d, %d",&(p
->min
),&(p
->del
),
309 &(p
->mapentries
),&(p
->quantvals
))!=4){
310 fprintf(stderr
,"5: syntax in %s in line:\t %s",filename
,line
);
315 if(sscanf(line
,"%ld",&(p
->fittotal
))!=1){
316 fprintf(stderr
,"6: syntax in %s in line:\t %s",filename
,line
);
320 find_seek_to(in
,"static const long _vq_pigeonmap_");
322 p
->pigeonmap
=_ogg_malloc(sizeof(long)*p
->mapentries
);
323 for(i
=0;i
<p
->mapentries
;i
++)
324 if(get_next_ivalue(in
,p
->pigeonmap
+i
)){
325 fprintf(stderr
,"out of data (pigeonmap) while reading codebook %s\n",filename
);
329 find_seek_to(in
,"static const long _vq_fitlist_");
331 p
->fitlist
=_ogg_malloc(sizeof(long)*p
->fittotal
);
332 for(i
=0;i
<p
->fittotal
;i
++)
333 if(get_next_ivalue(in
,p
->fitlist
+i
)){
334 fprintf(stderr
,"out of data (fitlist) while reading codebook %s\n",filename
);
338 find_seek_to(in
,"static const long _vq_fitmap_");
340 for(i
=0;i
<c
->dim
;i
++)pigeons
*=p
->quantvals
;
341 p
->fitmap
=_ogg_malloc(sizeof(long)*pigeons
);
342 for(i
=0;i
<pigeons
;i
++)
343 if(get_next_ivalue(in
,p
->fitmap
+i
)){
344 fprintf(stderr
,"out of data (fitmap) while reading codebook %s\n",filename
);
349 find_seek_to(in
,"static const long _vq_fitlength_");
351 p
->fitlength
=_ogg_malloc(sizeof(long)*pigeons
);
352 for(i
=0;i
<pigeons
;i
++)
353 if(get_next_ivalue(in
,p
->fitlength
+i
)){
354 fprintf(stderr
,"out of data (fitlength) while reading codebook %s\n",filename
);
364 quant_to_read
=_book_maptype1_quantvals(c
);
367 quant_to_read
=c
->entries
*c
->dim
;
371 /* load the quantized entries */
372 find_seek_to(in
,"static const long _vq_quantlist_");
374 c
->quantlist
=_ogg_malloc(sizeof(long)*quant_to_read
);
375 for(i
=0;i
<quant_to_read
;i
++)
376 if(get_next_ivalue(in
,c
->quantlist
+i
)){
377 fprintf(stderr
,"out of data while reading codebook %s\n",filename
);
381 /* load the lengthlist */
382 find_seek_to(in
,"_lengthlist");
384 c
->lengthlist
=_ogg_malloc(sizeof(long)*c
->entries
);
385 for(i
=0;i
<c
->entries
;i
++)
386 if(get_next_ivalue(in
,c
->lengthlist
+i
)){
387 fprintf(stderr
,"out of data while reading codebook %s\n",filename
);
394 vorbis_book_init_encode(b
,c
);
399 void spinnit(char *s
,int n
){
401 static long lasttime
=0;
403 struct timeval thistime
;
405 gettimeofday(&thistime
,NULL
);
406 test
=thistime
.tv_sec
*10+thistime
.tv_usec
/100000;
410 fprintf(stderr
,"%s%d ",s
,n
);
415 fprintf(stderr
,"| \r");
418 fprintf(stderr
,"/ \r");
421 fprintf(stderr
,"- \r");
424 fprintf(stderr
,"\\ \r");
431 void build_tree_from_lengths(int vals
, long *hist
, long *lengths
){
433 long *membership
=_ogg_malloc(vals
*sizeof(long));
434 long *histsave
=alloca(vals
*sizeof(long));
435 memcpy(histsave
,hist
,vals
*sizeof(long));
437 for(i
=0;i
<vals
;i
++)membership
[i
]=i
;
439 /* find codeword lengths */
440 /* much more elegant means exist. Brute force n^2, minimum thought */
442 int first
=-1,second
=-1;
445 spinnit("building... ",i
);
447 /* find the two nodes to join */
449 if(least
==-1 || hist
[j
]<=least
){
455 if((least
==-1 || hist
[j
]<=least
) && membership
[j
]!=first
){
457 second
=membership
[j
];
459 if(first
==-1 || second
==-1){
460 fprintf(stderr
,"huffman fault; no free branch\n");
465 least
=hist
[first
]+hist
[second
];
467 if(membership
[j
]==first
|| membership
[j
]==second
){
473 for(i
=0;i
<vals
-1;i
++)
474 if(membership
[i
]!=membership
[i
+1]){
475 fprintf(stderr
,"huffman fault; failed to build single tree\n");
479 /* for sanity check purposes: how many bits would it have taken to
480 encode the training set? */
485 bitsum
+=(histsave
[i
]-1)*lengths
[i
];
486 samples
+=histsave
[i
]-1;
490 fprintf(stderr
,"\rTotal samples in training set: %ld \n",samples
);
491 fprintf(stderr
,"\rTotal bits used to represent training set: %ld\n",
499 /* wrap build_tree_from_lengths to allow zero entries in the histogram */
500 void build_tree_from_lengths0(int vals
, long *hist
, long *lengths
){
502 /* pack the 'sparse' hit list into a dense list, then unpack
503 the lengths after the build */
506 long *lengthlist
=_ogg_calloc(vals
,sizeof(long));
507 long *newhist
=alloca(vals
*sizeof(long));
511 newhist
[upper
++]=hist
[i
];
514 fprintf(stderr
,"\rEliminating %d unused entries; %d entries remain\n",
518 build_tree_from_lengths(upper
,newhist
,lengthlist
);
523 lengths
[i
]=lengthlist
[upper
++];
530 void write_codebook(FILE *out
,char *name
,const static_codebook
*c
){
531 encode_aux_pigeonhole
*p
=c
->pigeon_tree
;
532 encode_aux_threshmatch
*t
=c
->thresh_tree
;
533 encode_aux_nearestmatch
*n
=c
->nearest_tree
;
536 /* save the book in C header form */
538 /* first, the static vectors, then the book structure to tie it together. */
541 long vals
=(c
->maptype
==1?_book_maptype1_quantvals(c
):c
->entries
*c
->dim
);
542 fprintf(out
,"static const long _vq_quantlist_%s[] = {\n",name
);
544 fprintf(out
,"\t%ld,\n",c
->quantlist
[j
]);
546 fprintf(out
,"};\n\n");
550 fprintf(out
,"static const long _vq_lengthlist_%s[] = {\n",name
);
551 for(j
=0;j
<c
->entries
;){
553 for(k
=0;k
<16 && j
<c
->entries
;k
++,j
++)
554 fprintf(out
,"%2ld,",c
->lengthlist
[j
]);
557 fprintf(out
,"};\n\n");
561 fprintf(out
,"static const float _vq_quantthresh_%s[] = {\n",name
);
562 for(j
=0;j
<t
->threshvals
-1;){
564 for(k
=0;k
<8 && j
<t
->threshvals
-1;k
++,j
++)
565 fprintf(out
,"%.5g, ",t
->quantthresh
[j
]);
568 fprintf(out
,"};\n\n");
571 fprintf(out
,"static const long _vq_quantmap_%s[] = {\n",name
);
572 for(j
=0;j
<t
->threshvals
;){
574 for(k
=0;k
<8 && j
<t
->threshvals
;k
++,j
++)
575 fprintf(out
,"%5ld,",t
->quantmap
[j
]);
578 fprintf(out
,"};\n\n");
580 fprintf(out
,"static const encode_aux_threshmatch _vq_auxt_%s = {\n",name
);
581 fprintf(out
,"\t(float *)_vq_quantthresh_%s,\n",name
);
582 fprintf(out
,"\t(long *)_vq_quantmap_%s,\n",name
);
583 fprintf(out
,"\t%d,\n",t
->quantvals
);
584 fprintf(out
,"\t%d\n};\n\n",t
->threshvals
);
589 for(i
=0;i
<c
->dim
;i
++)pigeons
*=p
->quantvals
;
592 fprintf(out
,"static const long _vq_pigeonmap_%s[] = {\n",name
);
593 for(j
=0;j
<p
->mapentries
;){
595 for(k
=0;k
<8 && j
<p
->mapentries
;k
++,j
++)
596 fprintf(out
,"%5ld, ",p
->pigeonmap
[j
]);
599 fprintf(out
,"};\n\n");
601 fprintf(out
,"static const long _vq_fitlist_%s[] = {\n",name
);
602 for(j
=0;j
<p
->fittotal
;){
604 for(k
=0;k
<8 && j
<p
->fittotal
;k
++,j
++)
605 fprintf(out
,"%5ld, ",p
->fitlist
[j
]);
608 fprintf(out
,"};\n\n");
610 fprintf(out
,"static const long _vq_fitmap_%s[] = {\n",name
);
613 for(k
=0;k
<8 && j
<pigeons
;k
++,j
++)
614 fprintf(out
,"%5ld, ",p
->fitmap
[j
]);
617 fprintf(out
,"};\n\n");
619 fprintf(out
,"static const long _vq_fitlength_%s[] = {\n",name
);
622 for(k
=0;k
<8 && j
<pigeons
;k
++,j
++)
623 fprintf(out
,"%5ld, ",p
->fitlength
[j
]);
626 fprintf(out
,"};\n\n");
628 fprintf(out
,"static const encode_aux_pigeonhole _vq_auxp_%s = {\n",name
);
629 fprintf(out
,"\t%g, %g, %d, %d,\n",
630 p
->min
,p
->del
,p
->mapentries
,p
->quantvals
);
632 fprintf(out
,"\t_vq_pigeonmap_%s,\n",name
);
634 fprintf(out
,"\t%ld,\n",p
->fittotal
);
635 fprintf(out
,"\t(long *)_vq_fitlist_%s,\n",name
);
636 fprintf(out
,"\t(long *)_vq_fitmap_%s,\n",name
);
637 fprintf(out
,"\t(long *)_vq_fitlength_%s\n};\n\n",name
);
643 fprintf(out
,"static const long _vq_ptr0_%s[] = {\n",name
);
646 for(k
=0;k
<8 && j
<n
->aux
;k
++,j
++)
647 fprintf(out
,"%6ld,",n
->ptr0
[j
]);
650 fprintf(out
,"};\n\n");
653 fprintf(out
,"static const long _vq_ptr1_%s[] = {\n",name
);
656 for(k
=0;k
<8 && j
<n
->aux
;k
++,j
++)
657 fprintf(out
,"%6ld,",n
->ptr1
[j
]);
660 fprintf(out
,"};\n\n");
663 fprintf(out
,"static const long _vq_p_%s[] = {\n",name
);
666 for(k
=0;k
<8 && j
<n
->aux
;k
++,j
++)
667 fprintf(out
,"%6ld,",n
->p
[j
]*c
->dim
);
670 fprintf(out
,"};\n\n");
673 fprintf(out
,"static const long _vq_q_%s[] = {\n",name
);
676 for(k
=0;k
<8 && j
<n
->aux
;k
++,j
++)
677 fprintf(out
,"%6ld,",n
->q
[j
]*c
->dim
);
680 fprintf(out
,"};\n\n");
682 fprintf(out
,"static const encode_aux_nearestmatch _vq_auxn_%s = {\n",name
);
683 fprintf(out
,"\t(long *)_vq_ptr0_%s,\n",name
);
684 fprintf(out
,"\t(long *)_vq_ptr1_%s,\n",name
);
685 fprintf(out
,"\t(long *)_vq_p_%s,\n",name
);
686 fprintf(out
,"\t(long *)_vq_q_%s,\n",name
);
687 fprintf(out
,"\t%ld, %ld\n};\n\n",n
->aux
,n
->aux
);
690 /* tie it all together */
692 fprintf(out
,"static const static_codebook %s = {\n",name
);
694 fprintf(out
,"\t%ld, %ld,\n",c
->dim
,c
->entries
);
695 fprintf(out
,"\t(long *)_vq_lengthlist_%s,\n",name
);
696 fprintf(out
,"\t%d, %ld, %ld, %d, %d,\n",
697 c
->maptype
,c
->q_min
,c
->q_delta
,c
->q_quant
,c
->q_sequencep
);
699 fprintf(out
,"\t(long *)_vq_quantlist_%s,\n",name
);
701 fprintf(out
,"\tNULL,\n");
704 fprintf(out
,"\t(encode_aux_nearestmatch *)&_vq_auxn_%s,\n",name
);
706 fprintf(out
,"\tNULL,\n");
708 fprintf(out
,"\t(encode_aux_threshmatch *)&_vq_auxt_%s,\n",name
);
710 fprintf(out
,"\tNULL,\n");
712 fprintf(out
,"\t(encode_aux_pigeonhole *)&_vq_auxp_%s,\n",name
);
714 fprintf(out
,"\tNULL,\n");
716 fprintf(out
,"\t0\n};\n\n");