tools/filetool/ftimpl.cpp

   1 #include "../common/pch.h"
   2 #include "filetool.h"
   3 #include "f_parser.h"
   4 #include "zlib.h"
   5
   6 feFileTool::feFileTool( int argc, TCHAR *argv[] )
   7 {
   8         mArgc = argc;
   9         mArgv = argv;
  10         mpRoot = NULL;
  11         mTableOffset = 0;
  12         mNumFiles = 0;
  13         mCurrentFile = 0;
  14 }
  15
  16 feFileTool::~feFileTool()
  17 {
  18         std::vector< feRegexp * >::iterator itIgnore;
  19         for ( itIgnore = mIgnoreList.begin(); itIgnore != mIgnoreList.end(); itIgnore++ )
  20                 delete ( *itIgnore );
  21         std::map< feRegexp *, feCStr >::iterator itRules;
  22         for ( itRules = mRulesList.begin(); itRules != mRulesList.end(); itRules++ )
  23                 delete ( *itRules ).first;
  24         if ( mpRoot )
  25                 delete mpRoot;
  26 }
  27
  28 int     feFileTool::run()
  29 {
  30         try {
  31
  32                 // parse command line
  33                 if ( mArgc <= 1 )
  34                 {
  35                         printUsage();
  36                         return 0;
  37                 }
  38
  39                 if ( !parseArgs() )
  40                         return false;
  41
  42                 if ( mCommand == cmdRebuild )
  43                         rebuild();
  44                 // else
  45                 // no commands were specified
  46         }
  47         catch ( feFileToolErr e )
  48         {
  49                 _tprintf( _T( "%s\n" ), e.msg().c_str() );
  50                 return -1;
  51         }
  52         return 0;
  53 }
  54
  55 void feFileTool::rebuild( void )
  56 {
  57         if ( mOutFile.empty() )
  58         {
  59                 _tprintf( _T( "rebuild command requires output filename. use -o option.\n" ) );
  60                 return;
  61         }
  62         loadIgnoreList();
  63         loadRulesList();
  64
  65         mpRoot = new file;
  66         mpRoot->blocktype = feFSBlockType_Folder;
  67         createFSStructure( mpRoot );
  68         _tprintf( _T( "total number of files: %d\n" ), mNumFiles );
  69
  70 #ifdef _DEBUG
  71         dbgPrintTree( mpRoot );
  72 #endif
  73
  74         _tprintf( _T( "opening '%s' for writing...\n" ), mOutFile.c_str() );
  75         FILE *fp = _tfopen( mOutFile.c_str(), _T( "w+b" ) );
  76         if ( !fp )
  77         {
  78                 _tprintf( _T( "cannot open file. exiting.\n" ) );
  79                 return;
  80         }
  81
  82         _tprintf( _T( "writing magic...\n" ) );
  83         char magic[] = "FEPAK";
  84         fwrite( magic, strlen( magic ), 1, fp );
  85         fwrite( &mTableOffset, sizeof( mTableOffset ), 1, fp );
  86
  87         mStartTime = (double)clock() / CLOCKS_PER_SEC;
  88
  89         mTotalSize = 0;
  90         processFiles( fp, mpRoot, "" );
  91         _tprintf( _T( "\n" ) );
  92
  93         _tprintf( _T( "approximated filesize = %.3lf mb\n" ), mTotalSize / ( 1024.0 * 1024.0 ) );
  94
  95         mTableOffset = ftell( fp );
  96         writeTable( fp, mpRoot );
  97         fseek( fp, (long)strlen( magic ), SEEK_SET );
  98         fwrite( &mTableOffset, sizeof( mTableOffset ), 1, fp );
  99         _tprintf( _T( "closing output file...\n" ) );
 100         fclose( fp );
 101         _tprintf( _T( "done!\n" ) );
 102 }
 103
 104 void feFileTool::printUsage( void )
 105 {
 106         _tprintf( _T( "usage: filetool.exe [options] command [command-options-and-arguments]\n" ) );
 107         _tprintf( _T( "  where options are -o, -c, etc.\n" ) );
 108         _tprintf( _T( "    (specify --help-options for a list of options)\n" ) );
 109         _tprintf( _T( "  where command are add, update, remove, etc.\n" ) );
 110         _tprintf( _T( "    (specify --help-commands for a list of commands)\n" ) );
 111         _tprintf( _T( "  where command-options-and-arguments depend on the specific command\n" ) );
 112         _tprintf( _T( "    (specify --help followed by a command name for command-specific help)\n" ) );
 113         _tprintf( _T( "  specify --help to recieve this message\n" ) );
 114         _tprintf( _T( "\nfe file processing tool v.0.1\n" ) );
 115 }
 116
 117 bool feFileTool::parseArgs( void )
 118 {
 119         for ( int i = 1; i < mArgc; i++ )
 120         {
 121                 // help
 122                 if ( !_tcsicmp( mArgv[i], _T( "--help" ) ) )
 123                 {
 124                         if ( i == mArgc-1 )
 125                         {
 126                                 printUsage();
 127                                 return false;
 128                         }
 129                         else
 130                         {
 131                                 i++;
 132                                 printCommandHelp( i );
 133                                 return false;
 134                         }
 135                 }
 136                 else if ( !_tcsicmp( mArgv[i], _T( "--help-options" ) ) )
 137                 {
 138                         printOptionsHelp();
 139                         return false;
 140                 }
 141                 else if ( !_tcsicmp( mArgv[i], _T( "--help-commands" ) ) )
 142                 {
 143                         printCommandsHelp();
 144                         return false;
 145                 }
 146                 // options
 147                 else if ( !_tcsnicmp( mArgv[i], _T( "-o" ), 2 ) )               // outfile
 148                 {
 149                         mOutFile = mArgv[i]+2;
 150                         if ( mOutFile.empty() )
 151                         {
 152                                 _tprintf( _T( "empty filename specified with -o\n" ) );
 153                         }
 154                 }
 155                 // commands
 156                 else if ( !_tcsicmp( mArgv[i], _T( "rebuild" ) ) )              // rebuild all
 157                 {
 158                         mCommand = cmdRebuild;
 159                         return true;
 160                 }
 161                 // error
 162                 else
 163                 {
 164                         _tprintf( _T( "unknown keyword: %s\n" ), mArgv[i] );
 165                         return false;
 166                 }
 167         }
 168         return true;
 169 }
 170
 171 void feFileTool::printOptionsHelp( void )
 172 {
 173         _tprintf( _T( "filetool global options (specified before command name) are:\n" ) );
 174         _tprintf( _T( "\t-o filename\t\tuse 'filename' as output file.\n" ) );
 175         _tprintf( _T( "\n(specify the --help global option for a list of other help options)\n" ) );
 176 }
 177
 178 void feFileTool::printCommandsHelp( void )
 179 {
 180         _tprintf( _T( "filetool commands are:\n" ) );
 181         _tprintf( _T( "\trebuild\t\trebuilds entire package from scratch.\n" ) );
 182         _tprintf( _T( "\n(specify the --help global option for a list of other help options)\n" ) );
 183 }
 184
 185 void feFileTool::printCommandHelp( int argIndex )
 186 {
 187         if ( !_tcsicmp( mArgv[argIndex], _T( "rebuild" ) ) )
 188         {
 189                 _tprintf( _T( "usage: filetool rebuild\n" ) );
 190         }
 191         _tprintf( _T( "(specify the --help global option for a list of other help options)\n" ) );
 192 }
 193
 194 void feFileTool::loadIgnoreList()
 195 {
 196         _tprintf( _T( "reading '.ftignore'...\n" ) );
 197         bool    expEof;
 198         try {
 199                 feCharParser p( ".ftignore", true );
 200                 for ( ;; )
 201                 {
 202                         expEof = true;
 203                         p.getToken();
 204                         expEof = false;
 205                         feRegexp *r = new feRegexp( p.token() );
 206                         mIgnoreList.push_back( r );
 207                         p.matchToken( ";" );
 208                 }
 209         }
 210         catch ( feParserException e )
 211         {
 212                 if ( e == feParserException::FILE_NOT_FOUND )
 213                 {
 214                         _tprintf( _T( "'.ftignore' does not exist. all contents will be processed.\n" ) );
 215                 }
 216                 else if ( e == feParserException::END_OF_FILE && false == expEof )
 217                 {
 218                         throw feFileToolErr( _T( "unexpected eof while reading '.ftignore'" ) );
 219                 }
 220                 else if ( e != feParserException::END_OF_FILE )
 221                         _tprintf( _T( "parser error: %s" ), feStr (e) );
 222         }
 223
 224         // add always-ignored items
 225         mIgnoreList.push_back( new feRegexp( "\\.ftignore$" ) );
 226         mIgnoreList.push_back( new feRegexp( "\\.ftrules$" ) );
 227         mIgnoreList.push_back( new feRegexp( "filetool\\.exe$" ) );
 228 }
 229
 230 void feFileTool::loadRulesList()
 231 {
 232         _tprintf( _T( "reading '.ftrules'...\n" ) );
 233         bool    expEof;
 234         try {
 235                 feCharParser p( ".ftrules", true );
 236                 for ( ;; )
 237                 {
 238                         expEof = true;
 239                         p.getToken();
 240                         feCStr re = p.token();
 241                         expEof = false;
 242                         p.getToken();
 243                         feRegexp *r = new feRegexp( re );
 244                         mRulesList[r] = p.token();
 245                         p.matchToken( ";" );
 246                 }
 247         }
 248         catch ( feParserException e )
 249         {
 250                 if ( e == feParserException::FILE_NOT_FOUND )
 251                 {
 252                         _tprintf( _T( "'.ftrules' does not exist. all files will be converted to compressed binaries ( feFSBlockType_BinaryCompressed ).\n" ) );
 253                         return;
 254                 }
 255                 else if ( e == feParserException::END_OF_FILE && false == expEof )
 256                 {
 257                         throw feFileToolErr( _T( "unexpected eof while reading '.ftrules'" ) );
 258                 }
 259                 else if ( e != feParserException::END_OF_FILE )
 260                         _tprintf( _T( "parser error: %s" ), feStr (e));
 261         }
 262 }
 263
 264 void feFileTool::createFSStructure( file *f )
 265 {
 266         if ( f == mpRoot )
 267                 _tprintf( _T( "creating filesystem structure...\n" ) );
 268     WIN32_FIND_DATA fd;
 269         memset( &fd, 0, sizeof( fd ) );
 270         HANDLE hFind = ::FindFirstFile( _T( "*.*" ), &fd );
 271         if ( hFind != INVALID_HANDLE_VALUE )
 272         {
 273                 do
 274                 {
 275                         feStr fname = fd.cFileName;
 276                         fname.tolower();
 277                         if ( !testIgnoreList( fname ) )
 278                         {
 279                                 if ( fname != _T( ".." ) && fname != _T( "." ) && _tcslen( fname ) <= 32 )
 280                                 {
 281                                         // add file
 282                                         mNumFiles++;
 283                                         file *nf = new file;
 284                                         strcpy( nf->name, feCStr( fname ).c_str() );
 285                                         nf->parent = f;
 286
 287                                         if ( fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY )
 288                                         {
 289                                                 // recurse
 290                                                 TCHAR rmbFolder[_MAX_PATH];
 291                                                 _tgetcwd( rmbFolder, _MAX_PATH );
 292                                                 _tchdir( fname );
 293                                                 createFSStructure( nf );
 294                                                 _tchdir( rmbFolder );
 295                                                 nf->blocktype = feFSBlockType_Folder;
 296                                         }
 297                                         else
 298                                         {
 299                                                 // find rule, default is 'c'
 300                                                 feCStr rule = findRule( fname );
 301                                                 if ( rule == "c" )
 302                                                         nf->blocktype = feFSBlockType_BinaryCompressed;
 303                                                 else if ( rule == "b" )
 304                                                         nf->blocktype = feFSBlockType_BinaryRawData;
 305                                                 else
 306                                                 {
 307                                                         feStr s;
 308                                                         s.printf( _T( "invalid rule: '%s'" ), rule.c_str() );
 309                                                         throw feFileToolErr( s );
 310                                                 }
 311                                                 // write filesize
 312                                                 if ( fd.nFileSizeHigh != 0 )
 313                                                 {
 314                                                         feStr s;
 315                                                         s.printf( _T( "file size is too big: '%s'" ), fname.c_str() );
 316                                                         throw feFileToolErr( s );
 317                                                 }
 318
 319                                                 nf->size = fd.nFileSizeLow;
 320                                         }
 321                                         f->children.push_back( nf );
 322                                 }
 323                         }
 324                 } while ( FindNextFile( hFind, &fd ) );
 325         }
 326 }
 327
 328 bool feFileTool::testIgnoreList( const feStr &s )
 329 {
 330         feCStr cs = s;
 331         std::vector< feRegexp * >::iterator it;
 332         for ( it = mIgnoreList.begin(); it != mIgnoreList.end(); it++ )
 333         {
 334                 char *str = new char[cs.size() + 1];
 335                 strcpy( str, cs.c_str() );
 336                 int r = regexec( ( *it )->getRegexp(), str );
 337                 delete[] str;
 338                 if ( r )
 339                         return true;
 340         }
 341         return false;
 342 }
 343
 344 feCStr feFileTool::findRule( const feStr &s )
 345 {
 346         feCStr cs = s;
 347         std::map< feRegexp *, feCStr >::iterator it;
 348         for ( it = mRulesList.begin(); it != mRulesList.end(); it++ )
 349         {
 350                 char *str = new char[cs.size() + 1];
 351                 strcpy( str, cs.c_str() );
 352                 int r = regexec( ( *it ).first->getRegexp(), str );
 353                 delete[] str;
 354                 if ( r )
 355                         return ( *it ).second;
 356         }
 357
 358         return _T( "c" );
 359 }
 360
 361 void feFileTool::dbgPrintTree( file *f, int indent )
 362 {
 363         int i;
 364         for ( i = 0; i < indent; i++ )
 365                 _tprintf( _T( "--" ) );
 366         if ( false == f->children.empty() )
 367                 _tprintf( _T( "[" ) );
 368         _tprintf( feStr( f->name ).c_str() );
 369         if ( false == f->children.empty() )
 370                 _tprintf( _T( "]" ) );
 371         _tprintf( _T( "\n" ) );
 372         for ( i = 0; i < (int)f->children.size(); i++ )
 373                 dbgPrintTree( f->children[i], indent+1 );
 374 }
 375
 376 void feFileTool::processFiles( FILE *fp, file *f, const feCStr &dir )
 377 {
 378         if ( f == mpRoot )
 379                 _tprintf( _T( "writing files...\n" ) );
 380         if ( f->blocktype != feFSBlockType_Folder )     // folder is dummy
 381         {
 382                 f->offset = ftell( fp );
 383
 384                 uchar *buffer = new uchar[f->size];
 385                 feCStr s;
 386                 // maxname is 32 chars, terminating 0 is not included
 387                 // copy string and append with 0
 388                 char str[33];
 389                 memset( str, 0, 33 );
 390                 memcpy( str, f->name, 32 );
 391                 s.printf( "%s%s", dir.c_str(), str );
 392                 FILE *in = fopen( s.c_str(), "rb" );
 393                 if ( !in )
 394                 {
 395                         _tprintf( _T( "*** warning: cannot open file '%s', filling with zeros.\n" ), feStr( s.c_str() ).c_str() );
 396                         _tprintf( _T( "*** hint: file name may contain non-OEM characters.\n" ) );
 397                         memset( buffer, 0, f->size );
 398                 }
 399                 else
 400                 {
 401                         memset( buffer, 0, f->size );
 402                         fread( buffer, f->size, 1, in );
 403                         fclose( in );
 404                 }
 405
 406                 if ( f->blocktype == feFSBlockType_BinaryRawData )
 407                 {
 408                         fwrite( buffer, f->size, 1, fp );
 409                         mTotalSize += f->size;
 410                         f->comprsize = f->size;
 411                 }
 412                 else if ( f->blocktype == feFSBlockType_BinaryCompressed )
 413                 {
 414                         f->comprsize = (ulong)(f->size * 1.3);
 415                         uchar *compr = new uchar[f->comprsize];
 416                         compress2( compr, &f->comprsize, buffer, f->size, 9 );
 417                         fwrite( compr, f->comprsize, 1, fp );
 418                         mTotalSize += f->comprsize;
 419                         delete[] compr;
 420                 }
 421                 else
 422                         __asm int 3;    // break execution: something wrong
 423
 424                 delete[] buffer;
 425
 426                 double elapsedtime = (double)clock() / CLOCKS_PER_SEC - mStartTime;
 427                 double timeperfile = elapsedtime / ( mCurrentFile ? mCurrentFile : 1 );
 428                 double totaltime = timeperfile * mNumFiles;
 429                 timeperfile /= 60.0;
 430                 elapsedtime /= 60.0;
 431                 totaltime /= 60.0;
 432                 printf( "%d%% (%d/%d); elapsed time %.2lfm, estimated time left: %.2lfm, avg time per file: %.5lfm    \r"
 433                         , (int)( 100.0 * ( mCurrentFile + 1 ) / mNumFiles )
 434                         , mCurrentFile+1, mNumFiles
 435                         , elapsedtime
 436                         , totaltime - elapsedtime
 437                         , timeperfile );
 438                 mCurrentFile++;
 439         }
 440         else
 441         {
 442                 if ( f != mpRoot )
 443                         mCurrentFile++;
 444                 for ( size_t i = 0; i < f->children.size(); i++ )
 445                 {
 446                         feCStr newDir = f == mpRoot ? "" : dir + feCStr(f->name) + feCStr("/");
 447                         processFiles( fp, f->children[i], newDir );
 448                 }
 449         }
 450 }
 451
 452 void feFileTool::writeTable( FILE *fp, file *f )
 453 {
 454         if ( f == mpRoot )
 455                 _tprintf( _T( "writing file table...\n" ) );
 456         fwrite( f, 48, 1, fp );
 457         if ( f->blocktype == feFSBlockType_Folder )
 458         {
 459                 ulong nc = ( ulong )f->children.size();
 460                 fwrite( &nc, sizeof( nc ), 1, fp );
 461                 for ( ulong i = 0; i < nc; i++  )
 462                         writeTable( fp, f->children[i] );
 463         }
 464 }
 465