ryzom/tools/occ2huff/occ2huff.cpp

   1 // Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
   2 // Copyright (C) 2010  Winch Gate Property Limited
   3 //
   4 // This program is free software: you can redistribute it and/or modify
   5 // it under the terms of the GNU Affero General Public License as
   6 // published by the Free Software Foundation, either version 3 of the
   7 // License, or (at your option) any later version.
   8 //
   9 // This program is distributed in the hope that it will be useful,
  10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 // GNU Affero General Public License for more details.
  13 //
  14 // You should have received a copy of the GNU Affero General Public License
  15 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17
  18
  19
  20 #include "nel/misc/types_nl.h"
  21 #include "nel/misc/debug.h"
  22 #include "nel/misc/file.h"
  23
  24 #include "game_share/huffman.h"
  25
  26 #include <string>
  27 #include <map>
  28 #include <fstream>
  29
  30 using namespace std;
  31 using namespace NLMISC;
  32
  33
  34 /**
  35  * CStringInfos
  36  * \author Stephane Coutelas
  37  * \author Nevrax France
  38  * \date 2002
  39  */
  40 struct CStringInfos
  41 {
  42         /// string id
  43         string Id;
  44
  45         /// string
  46         string Str;
  47
  48         /// occurence of the string
  49         uint32 Occurence;
  50
  51         /**
  52          * Default constructor
  53          */
  54         CStringInfos() : Occurence(1) { }
  55
  56 };
  57
  58
  59
  60 //-----------------------------------------------
  61 //      main
  62 //
  63 //-----------------------------------------------
  64 sint main( sint argc, char ** argv )
  65 {
  66         uint currentVersion = 1;
  67
  68         map<string,CStringInfos> base;
  69
  70         if( argc < 3 )
  71         {
  72                 printf("Create a file associating a string id with a string and its Huffman code\n\n");
  73                 printf("OCC2HUFF <string file> [<string file> ...] <occ file>\n");
  74                 return 1;
  75         }
  76
  77         // open the id string association file(s)
  78         sint i;
  79         for( i = 1; i < argc - 1; i++ )
  80         {
  81                 printf("Reading string association file '%s'...\n",argv[i]);
  82
  83                 ifstream input1(argv[i], ios::in);
  84                 if( !input1.is_open() )
  85                 {
  86                         nlwarning("can't open the file %s",argv[i]);
  87                         return 1;
  88                 }
  89
  90                 // read the tokens and create the string infos
  91                 while( !input1.eof() )
  92                 {
  93                         // read a line
  94                         string line;
  95                         getline(input1,line,'\n');
  96
  97                         // test the line ( there must be at least 2 '"',remove comments if exist )
  98                         sint32 idx = line.find_first_of("#");
  99                         bool hasComments = false;
 100                         if( idx != -1 )
 101                         {
 102                                 line = line.substr(0,idx);
 103                                 hasComments = true;
 104                         }
 105                         if( line.size() == 0 )
 106                         {
 107                                 continue;
 108                         }
 109                         if( line.find_first_of("\"") == -1 )
 110                         {
 111                                 if( !hasComments )
 112                                 {
 113                                         nlwarning("Missing string value in the string '%s'",line.c_str());
 114                                         return 1;
 115                                 }
 116                         }
 117                         if( line.find_first_of("\"") == line.find_last_of("\"") )
 118                         {
 119                                 if( !hasComments )
 120                                 {
 121                                         nlwarning("Missing a delimiter \" in the string '%s'",line.c_str());
 122                                         return 1;
 123                                 }
 124                         }
 125
 126                         // extract string id and string
 127                         idx = line.find_first_of(" \t");
 128                         if( idx != -1 )
 129                         {
 130                                 CStringInfos si;
 131                                 si.Id = line.substr(0,idx);
 132
 133                                 sint32 startIdx = line.find_first_of("\"");
 134                                 sint32 endIdx = line.find_last_of("\"");
 135                                 si.Str = line.substr(startIdx+1,endIdx-startIdx-1);
 136
 137                                 // add string infos
 138                                 map<string,CStringInfos>::iterator itStr = base.find( si.Id );
 139                                 if( itStr == base.end() )
 140                                 {
 141                                         base.insert( make_pair(si.Id,si) );
 142                                 }
 143                                 else
 144                                 {
 145                                         nlwarning("The string %s already exists !",si.Id.c_str());
 146                                 }
 147                         }
 148                 }
 149                 input1.close();
 150         }
 151
 152         // open the id occurence association file
 153         string occfilename = argv[argc-1];
 154         printf("Reading occurence file '%s'...\n",occfilename.c_str());
 155         ifstream input2(occfilename.c_str(), ios::in);
 156         if( !input2.is_open() )
 157         {
 158                 nlwarning("Can't open the file %s, set all occurences to 1",argv[argc-1]);
 159         }
 160         else
 161         {
 162                 // read the tokens and update the string infos with occurences
 163                 while( !input2.eof() )
 164                 {
 165                         // read a line
 166                         string line;
 167                         getline(input2,line,'\n');
 168
 169                         // test the line
 170                         sint32 idx = line.find_first_of("#");
 171                         if( idx != string::npos )
 172                         {
 173                                 line = line.substr(0,idx);
 174                         }
 175                         if( line.size() == 0 )
 176                         {
 177                                 continue;
 178                         }
 179                         char * buffer = new char[line.size()+1];
 180                         strcpy(buffer,line.c_str());
 181
 182                         // extract string id and occurence
 183                         char * token;
 184                         string stoken;
 185                         token = strtok(buffer," \t");
 186                         if( token != NULL )
 187                         {
 188                                 stoken = string( token );
 189                                 map<string,CStringInfos>::iterator itStr = base.find( stoken );
 190                                 if( itStr != base.end() )
 191                                 {
 192                                         token = strtok(NULL," \t");
 193                                         (*itStr).second.Occurence = atoi( token );
 194                                         if( (*itStr).second.Occurence == 0 )
 195                                         {
 196                                                 nlwarning("The occurence of string '%s' is 0 (problem with occurence ?: '%s'), set it to 1",(*itStr).second.Str.c_str(),token);
 197                                                 (*itStr).second.Occurence = 1;
 198                                         }
 199                                 }
 200                                 else
 201                                 {
 202                                         nlwarning("The string '%s' is in the .occ but in the txt files!",token);
 203                                 }
 204                         }
 205                         delete buffer;
 206                 }
 207                 input2.close();
 208
 209                 ofstream output2(occfilename.c_str(), ios::app);
 210                 if (output2.is_open())
 211                 {
 212                         map<string,CStringInfos>::iterator itBase;
 213                         for( itBase = base.begin(); itBase != base.end(); ++itBase )
 214                         {
 215                                 if((*itBase).second.Occurence == 0)
 216                                 {
 217                                         output2 << (*itBase).first << " 1" <<endl;
 218                                         (*itBase).second.Occurence = 1;
 219                                 }
 220                         }
 221                         output2.close ();
 222                 }
 223         }
 224
 225         // build the Huffman tree
 226         printf("Building Huffman tree...\n");
 227         CHuffman huff;
 228         map<string,CStringInfos>::iterator itBase;
 229         for( itBase = base.begin(); itBase != base.end(); ++itBase )
 230         {
 231                 huff.add( (*itBase).first,(*itBase).second.Occurence );
 232         }
 233         huff.build();
 234
 235
 236         // open the output file
 237         string outputFileName = "chat_static.cdb";
 238         COFile output( outputFileName );
 239
 240         // save id|string|occurence
 241         printf("Writing binary file '%s'...\n",outputFileName.c_str());
 242         output.serialVersion(currentVersion);
 243         uint32 count = base.size();
 244         output.serial( count );
 245         vector<bool> code;
 246         for( itBase = base.begin(); itBase != base.end(); ++itBase )
 247         {
 248                 if( (*itBase).second.Occurence > 0 )
 249                 {
 250                         huff.getCode( (*itBase).first, code );
 251                         output.serial( (*itBase).second.Id );
 252                         output.serial( (*itBase).second.Str );
 253                         output.serial( (*itBase).second.Occurence );
 254                         output.serialCont( code );
 255                 }
 256         }
 257
 258         // TEST
 259         /*printf("Writing debug text file...\n");
 260         FILE * outputTest = nlfopen("chat_static_base_test.log","wt");
 261         for( itBase = base.begin(); itBase != base.end(); ++itBase )
 262         {
 263                 fprintf(outputTest,"id: %s  str: %s  occ: %d\n",(*itBase).second.Id.c_str(),(*itBase).second.Str.c_str(), (*itBase).second.Occurence );
 264         }
 265         fclose(outputTest);*/
 266
 267         printf("Process complete.\n");
 268
 269         return 0;
 270 }