nelns/admin_executor_service/log_report.cpp

   1 // NeLNS - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
   2 // Copyright (C) 2010  Winch Gate Property Limited
   3 //
   4 // This source file has been modified by the following contributors:
   5 // Copyright (C) 2011  Matt RAYKOWSKI (sfb) <matt.raykowski@gmail.com>
   6 //
   7 // This program is free software: you can redistribute it and/or modify
   8 // it under the terms of the GNU Affero General Public License as
   9 // published by the Free Software Foundation, either version 3 of the
  10 // License, or (at your option) any later version.
  11 //
  12 // This program is distributed in the hope that it will be useful,
  13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 // GNU Affero General Public License for more details.
  16 //
  17 // You should have received a copy of the GNU Affero General Public License
  18 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 #include "log_report.h"
  21 #include <functional>
  22 #include <limits>
  23 #include "nel/misc/common.h"
  24 #include "nel/misc/displayer.h"
  25 #include "nel/misc/file.h"
  26 #include "nel/misc/path.h"
  27 #include "nel/misc/variable.h"
  28
  29 using namespace NLMISC;
  30 using namespace std;
  31
  32
  33 CVariable<string> LogPath( "LogReport","LogPath", "Path of the log files", ".", 0, true );
  34
  35 const uint MAX_LOG_LINE_SIZE = 1024;
  36 //nlctassert(MAX_LOG_LINE_SIZE>0);
  37
  38 enum TLogLineHeader { LHDate, LHTime, LHType, LHThread, LHService, LHCodeFile, LHCodeLine, LHSeparator, LH_NB_FIELDS };
  39
  40
  41 ///
  42 bool isLogFile( const std::string& filename )
  43 {
  44         uint len = (uint)filename.size();
  45         return (len >= 4 ) && (filename.substr( len-4 ) == ".log");
  46 }
  47
  48 ///
  49 inline bool     isNumberChar( char c )
  50 {
  51         return (c >= '0') && (c <= '9');
  52 }
  53
  54 ///
  55 void sortLogFiles( vector<std::string>& filenames )
  56 {
  57         uint i;
  58         for ( i=0; i!=filenames.size(); ++i )
  59         {
  60                 // Ensure that a log file without number comes *after* the ones with a number
  61                 string name = string(filenames[i]);
  62                 string::size_type dotpos = name.find_last_of('.');
  63                 if ( (dotpos!=string::npos) && (dotpos > 2) )
  64                 {
  65                         if ( ! (isNumberChar(name[dotpos-1]) && isNumberChar(name[dotpos-2]) && isNumberChar(name[dotpos-3])) )
  66                         {
  67                                 name = name.substr( 0, dotpos ) + "ZZZ" + name.substr( dotpos );
  68                                 filenames[i] = name.c_str();
  69                         }
  70                 }
  71         }
  72         sort( filenames.begin(), filenames.end() );
  73         for ( i=0; i!=filenames.size(); ++i )
  74         {
  75                 // Set the original names back
  76                 string name = filenames[i];
  77                 string::size_type tokenpos = name.find( "ZZZ." );
  78                 if ( tokenpos != string::npos )
  79                 {
  80                         name = name.substr( 0, tokenpos ) + name.substr( tokenpos + 3 );
  81                         filenames[i] = name.c_str();
  82                 }
  83         }
  84 }
  85
  86 void    CMakeLogTask::setLogPath(const std::string & logPath)
  87 {
  88         _LogPaths.resize( 1 );
  89         _LogPaths[0] = logPath;
  90 }
  91
  92 void    CMakeLogTask::setLogPaths(const std::vector<std::string>& logPaths)
  93 {
  94         _LogPaths = logPaths;
  95 }
  96
  97 void    CMakeLogTask::setLogPathToDefault()
  98 {
  99         setLogPath( LogPath.get() );
 100 }
 101
 102 /*
 103  *
 104  */
 105 CMakeLogTask::~CMakeLogTask()
 106 {
 107         if ( _Thread ) // implies && _OutputLogReport
 108         {
 109                 if ( ! _Complete )
 110                 {
 111                         pleaseStop();
 112                         _Thread->wait();
 113                 }
 114                 clear();
 115         }
 116 }
 117
 118
 119 /*
 120  *
 121  */
 122 void    CMakeLogTask::start()
 123 {
 124         if ( _Thread )
 125         {
 126                 if ( _Complete )
 127                         clear();
 128                 else
 129                         return;
 130         }
 131         _Stopping = false;
 132         _Complete = false;
 133         _Thread = NLMISC::IThread::create( this );
 134         _OutputLogReport = new CLogReport();
 135         _Thread->start();
 136 }
 137
 138
 139 /*
 140  *
 141  */
 142 void    CMakeLogTask::clear()
 143 {
 144         if (_Thread)
 145         {
 146                 delete _Thread;
 147                 _Thread = NULL;
 148         }
 149         if (_OutputLogReport)
 150         {
 151                 delete _OutputLogReport;
 152                 _OutputLogReport = NULL;
 153         }
 154 }
 155
 156 /*
 157  *
 158  */
 159 void    CMakeLogTask::terminateTask()
 160 {
 161         if (!_Thread) // _Thread _implies _OutputLogReport
 162                 return;
 163
 164         pleaseStop();
 165         _Thread->wait();
 166
 167         clear();
 168 }
 169
 170 //
 171 bool isOfLogDotLogFamily( const std::string& filename )
 172 {
 173         return ((filename == "log.log") ||
 174                          ((filename.size() == 10) &&
 175                           (filename.substr( 0, 3 ) == "log") &&
 176                           isNumberChar(filename[3]) && isNumberChar(filename[4]) && isNumberChar(filename[5]) &&
 177                           (filename.substr( 6, 4 ) == ".log")) );
 178 }
 179
 180
 181 enum TVersionTargetMode { TTMAll, TTMMatchAllV, TTMMatchExactV, TTMMatchGreaterV, TTMMatchLowerV } targetMode;
 182 const uint CurrentVersion = std::numeric_limits<uint>::max();
 183
 184 // Return true and logVersion, or false if not a log with version
 185 bool getLogVersion( const std::string& filename, uint& logVersion )
 186 {
 187         uint len = (uint)filename.size();
 188         if ( (len > 4) && (filename.substr( len-4 ) == ".log") )
 189         {
 190                 if ( filename.substr(0, 3) == "log" )
 191                 {
 192                         if ( (len == 7) ||
 193                                  ((len == 10) && (isNumberChar(filename[3]) && isNumberChar(filename[4]) && isNumberChar(filename[5]))) )
 194                         {
 195                                 logVersion = CurrentVersion;
 196                                 return true;
 197                         }
 198                 }
 199                 else if ( filename[0] == 'v' )
 200                 {
 201                         string::size_type p = filename.find( "_", 1 );
 202                         if ( p != string::npos )
 203                         {
 204                                 if ( (len == p + 8) ||
 205                                          ((len == p + 11) && (isNumberChar(filename[p+4]) && isNumberChar(filename[p+5]) && isNumberChar(filename[p+6]))) )
 206                                 {
 207                                         NLMISC::fromString( filename.substr( 1, p-1 ), logVersion );
 208                                         return true;
 209                                 }
 210                         }
 211                 }
 212         }
 213         return false;
 214 }
 215
 216 // Assumes filename is .log file
 217 bool matchLogTarget( const std::string& filename, TVersionTargetMode targetMode, uint targetVersion )
 218 {
 219         if ( targetMode == TTMAll )
 220                 return true;
 221
 222         uint version;
 223
 224         // Get version or exclude non-standard log files
 225         if ( ! getLogVersion( filename, version ) )
 226                 return false;
 227
 228         // Exclude non-matching version
 229         switch ( targetMode )
 230         {
 231         case TTMMatchExactV:
 232                 return (version == targetVersion); // break;
 233         case TTMMatchGreaterV:
 234                 return (version >= targetVersion); // break;
 235         case TTMMatchLowerV:
 236                 return (version <= targetVersion); // break;
 237         default: // TTMMatchAllV
 238                 return true;
 239         }
 240 }
 241
 242 /*
 243  *
 244  */
 245 void    CMakeLogTask::run()
 246 {
 247         // Parse log target
 248         uint targetVersion = CurrentVersion;
 249         uint lts = (uint)_LogTarget.size();
 250         if ( _LogTarget.empty() || (_LogTarget == "v") )
 251         {
 252                 targetMode = TTMMatchExactV;
 253         }
 254         else if ( _LogTarget == "v*" )
 255         {
 256                 targetMode = TTMMatchAllV;
 257         }
 258         else if ( _LogTarget == "*" )
 259         {
 260                 targetMode = TTMAll;
 261         }
 262         else if ( (lts > 1) && (_LogTarget[0] == 'v') )
 263         {
 264                 uint additionalChars = 1;
 265                 if ( _LogTarget[lts-1] == '+' )
 266                         targetMode = TTMMatchGreaterV;
 267                 else if ( _LogTarget[lts-1] == '-' )
 268                         targetMode = TTMMatchLowerV;
 269                 else
 270                 {
 271                         targetMode = TTMMatchExactV;
 272                         additionalChars = 0;
 273                 }
 274
 275                 NLMISC::fromString( _LogTarget.substr( 1, lts-additionalChars-1 ), targetVersion );
 276         }
 277         else
 278         {
 279                 nlwarning( "Invalid log target argument: %s", _LogTarget.c_str() );
 280                 _Complete = true;
 281                 return;
 282         }
 283
 284         // Get log files and sort them
 285         vector<string> filenames;
 286         vector<string> filenamesOfPath;
 287         for ( vector<string>::const_iterator ilf=_LogPaths.begin(); ilf!=_LogPaths.end(); ++ilf )
 288         {
 289                 string path = (*ilf);
 290                 if ( (! path.empty()) && (path[path.size()-1]!='/') )
 291                         path += "/";
 292                 filenamesOfPath.clear();
 293                 CPath::getPathContent( path, false, false, true, filenamesOfPath, NULL, true );
 294                 vector<string>::iterator ilf2 = partition( filenamesOfPath.begin(), filenamesOfPath.end(), isLogFile );
 295                 filenamesOfPath.erase( ilf2, filenamesOfPath.end() );
 296                 sortLogFiles( filenamesOfPath );
 297                 filenames.insert( filenames.end(), filenamesOfPath.begin(), filenamesOfPath.end() );
 298         }
 299
 300         // Analyse log files
 301         _OutputLogReport->reset();
 302         uint nbLines = 0;
 303         char line [MAX_LOG_LINE_SIZE];
 304
 305         uint nbSkippedFiles = 0;
 306         for ( vector<string>::const_iterator ilf=filenames.begin(); ilf!=filenames.end(); ++ilf )
 307         {
 308                 string shortname = CFile::getFilename( *ilf );
 309
 310                 // Filter log files based on filename before opening them
 311                 if ( ! matchLogTarget( shortname, targetMode, targetVersion ) )
 312                 {
 313                         ++nbSkippedFiles;
 314                         continue;
 315                 }
 316
 317                 nlinfo( "Processing %s (%u/%u)", (*ilf).c_str(), ilf-filenames.begin(), filenames.size() );
 318                 CIFile logfile;
 319                 if ( logfile.open( *ilf, true ) )
 320                 {
 321                         _OutputLogReport->setProgress( (uint)(ilf-filenames.begin()), (uint)filenames.size() );
 322                         while ( ! logfile.eof() )
 323                         {
 324                                 logfile.getline( line, MAX_LOG_LINE_SIZE );
 325                                 line[MAX_LOG_LINE_SIZE-1] = '\0'; // force valid end of line
 326                                 _OutputLogReport->pushLine( line );
 327                                 ++nbLines;
 328
 329                                 if ( isStopping() )
 330                                         return;
 331                         }
 332                 }
 333         }
 334         nlinfo( "%u lines processed", nbLines );
 335         if ( nbSkippedFiles != 0 )
 336                 nlinfo( "%u log files skipped, not matching target %s", nbSkippedFiles, _LogTarget.c_str() );
 337         _Complete = true;
 338 }
 339
 340
 341 /*
 342  * Add a log line to the report tree
 343  */
 344 void    CLogReport::pushLine( const std::string& line, NLMISC::CLog::TLogType onlyType, bool countOtherTypes )
 345 {
 346         // Ignore session title
 347         if ( (line.size() > 14) && (line.substr( 0, 14 ) == "Log Starting [") )
 348                 return;
 349
 350         // Decode standard log line
 351         vector<string> lineTokens;
 352         explode( line, string(" "), lineTokens );
 353
 354         if ( lineTokens.size() < LH_NB_FIELDS )
 355                 return;
 356
 357         // Filter log type
 358         if ( onlyType != CLog::LOG_UNKNOWN )
 359         {
 360                 if ( lineTokens[LHType] != IDisplayer::logTypeToString( onlyType ) )
 361                 {
 362                         if ( countOtherTypes )
 363                                 storeLine( lineTokens, true );
 364                         return;
 365                 }
 366         }
 367
 368         // Store
 369         storeLine( lineTokens, false );
 370 }
 371
 372
 373 /*
 374  *
 375  */
 376 void    CLogReportNode::storeLine( const std::vector<std::string>& lineTokens, bool mainCountOnly )
 377 {
 378         // Get service name from "[machine/]serviceName-serviceId"
 379         string service = lineTokens[LHService];
 380         string::size_type p = service.find( '/' );
 381         if ( p != string::npos )
 382                 service = service.substr( p+1 );
 383         p = service.find( '-' );
 384         if ( p != string::npos )
 385                 service = service.substr( 0, p );
 386
 387         // Store to appropriate child
 388         CLogReportLeaf *child = getChild( service );
 389         if ( ! child )
 390                 child = addChild( service );
 391         child->storeLine( lineTokens, mainCountOnly );
 392 }
 393
 394
 395 /*
 396  *
 397  */
 398 void    CLogReportLeaf::storeLine( const std::vector<std::string>& lineTokens, bool mainCountOnly )
 399 {
 400         if ( ! mainCountOnly )
 401         {
 402                 // Build key from "codeFile codeLine"
 403                 string key = lineTokens[LHCodeFile] + ":" + lineTokens[LHCodeLine];
 404                 _LogLineInfo[key].addAnOccurence( lineTokens );
 405         }
 406         ++_Counts[lineTokens[LHType]];
 407         ++_TotalLines;
 408 }
 409
 410
 411 /*
 412  *
 413  */
 414 void    CLogLineInfo::addAnOccurence( const std::vector<std::string>& lineTokens )
 415 {
 416         if ( NbOccurences == 0 )
 417         {
 418                 for ( uint i=LH_NB_FIELDS; i<lineTokens.size(); ++i )
 419                 {
 420                         if ( i != LH_NB_FIELDS )
 421                                 SampleLogText += " ";
 422                         SampleLogText += lineTokens[i];
 423                 }
 424         }
 425         ++NbOccurences;
 426 }
 427
 428
 429 /*
 430  *
 431  */
 432 uint    CLogReportLeaf::getNbTotalLines( NLMISC::CLog::TLogType logType )
 433 {
 434         return (logType==NLMISC::CLog::LOG_UNKNOWN) ? _TotalLines : _Counts[NLMISC::IDisplayer::logTypeToString( logType )];
 435 }
 436
 437
 438 /*
 439  * Get results for a service
 440  */
 441 void    CLogReport::reportByService( const std::string& service, NLMISC::CLog *targetLog )
 442 {
 443         ILogReport *child = getChild( service );
 444         if ( child )
 445         {
 446                 child->report( targetLog, true );
 447         }
 448         else
 449         {
 450                 targetLog->displayNL( "Nothing found for service %s", service.c_str() );
 451         }
 452 }
 453
 454
 455 /*
 456  * Get results for a service (all distinct lines, sorted by occurence)
 457  */
 458 void    CLogReportLeaf::report( NLMISC::CLog *targetLog, bool )
 459 {
 460         // Sort it
 461         typedef multimap< uint, pair< string, const CLogLineInfo * >, std::greater<uint> > CSortedByOccurenceLogLineInfoMap;
 462         CSortedByOccurenceLogLineInfoMap sortedByOccurence;
 463         for ( CLogLineInfoMap::const_iterator it=_LogLineInfo.begin(); it!=_LogLineInfo.end(); ++it )
 464         {
 465                 const string &key = (*it).first;
 466                 const CLogLineInfo& info = (*it).second;
 467                 sortedByOccurence.insert( make_pair( info.NbOccurences, make_pair( key, &info ) ) );
 468         }
 469
 470         // Display it
 471         for ( CSortedByOccurenceLogLineInfoMap::const_iterator iso=sortedByOccurence.begin(); iso!=sortedByOccurence.end(); ++iso )
 472         {
 473                 const string &key = (*iso).second.first;
 474                 const CLogLineInfo& info = *((*iso).second.second);
 475                 targetLog->displayRawNL( "%s %6u %s : %s", _Service.c_str(), info.NbOccurences, key.c_str(), info.SampleLogText.c_str() );
 476         }
 477 }
 478
 479
 480 /*
 481  * Return the number of lines displayed
 482  */
 483 uint    CLogReportLeaf::reportPart( uint beginIndex, uint maxNbLines, NLMISC::CLog *targetLog )
 484 {
 485         uint i = 0;
 486         CLogLineInfoMap::const_iterator it;
 487         for ( it=_LogLineInfo.begin(); it!=_LogLineInfo.end(); ++it )
 488         {
 489                 if ( i >= beginIndex )
 490                 {
 491                         if ( i >= maxNbLines )
 492                                 return i - beginIndex;
 493
 494                         const string &key = (*it).first;
 495                         const CLogLineInfo& info = (*it).second;
 496                         targetLog->displayRawNL( "%s %6u %s : %s", _Service.c_str(), info.NbOccurences, key.c_str(), info.SampleLogText.c_str() );
 497                 }
 498                 ++i;
 499         }
 500         return i - beginIndex;
 501 }
 502
 503
 504 /*
 505  * Get summary of results
 506  */
 507 void    CLogReportNode::report( NLMISC::CLog *targetLog, bool displayDetailsPerService )
 508 {
 509         uint nb1Sum=0, nb2Sum=0;
 510         for ( std::vector<CLogReportLeaf*>::const_iterator it=_Children.begin(); it!=_Children.end(); ++it )
 511         {
 512                 CLogReportLeaf *pt = (*it);
 513
 514                 // Get distinct warnings
 515                 uint nb1 = pt->getNbDistinctLines();
 516                 nb1Sum += nb1;
 517
 518                 // Get total warnings, info... but filter out lines with no header
 519                 uint sumTotalLinesNotUnknown = 0;
 520                 uint nbTotalLines [CLog::LOG_UNKNOWN];
 521                 for ( uint i=CLog::LOG_ERROR; i!=CLog::LOG_UNKNOWN; ++i )
 522                 {
 523                         nbTotalLines[i] = pt->getNbTotalLines( (CLog::TLogType)i );
 524                         sumTotalLinesNotUnknown += nbTotalLines[i];
 525                 }
 526                 if ( sumTotalLinesNotUnknown != 0 )
 527                 {
 528                         targetLog->displayRawNL( "%s: \t%u distinct WRN, %u total WRN, %u INF, %u DBG, %u STT, %u AST, %u ERR, %u TOTAL",
 529                                 pt->service().c_str(), nb1, nbTotalLines[CLog::LOG_WARNING],
 530                                 nbTotalLines[CLog::LOG_INFO], nbTotalLines[CLog::LOG_DEBUG],
 531                                 nbTotalLines[CLog::LOG_STAT], nbTotalLines[CLog::LOG_ASSERT],
 532                                 nbTotalLines[CLog::LOG_ERROR], pt->getNbTotalLines( CLog::LOG_UNKNOWN ) );
 533                         nb2Sum += nbTotalLines[CLog::LOG_WARNING];
 534                 }
 535         }
 536         targetLog->displayRawNL( "=> %u distinct, %u total WRN (%u pages)", nb1Sum, nb2Sum, nb1Sum / NB_LINES_PER_PAGE + 1 );
 537
 538         if ( displayDetailsPerService )
 539         {
 540                 for ( std::vector<CLogReportLeaf*>::const_iterator it=_Children.begin(); it!=_Children.end(); ++it )
 541                 {
 542                         (*it)->report( targetLog, true );
 543                 }
 544         }
 545 }
 546
 547
 548 /*
 549  * Get partial results (pageNum>=1)
 550  */
 551 void    CLogReportNode::reportPage( uint pageNum, NLMISC::CLog *targetLog )
 552 {
 553         if ( _Children.empty() )
 554         {
 555                 targetLog->displayRawNL( "[END OF LOG]" );
 556                 return;
 557         }
 558
 559         uint beginIndex = pageNum * NB_LINES_PER_PAGE;
 560         uint lineCounter = 0, prevLineCounter;
 561         for ( std::vector<CLogReportLeaf*>::const_iterator it=_Children.begin(); it!=_Children.end(); ++it )
 562         {
 563                 CLogReportLeaf *pt = (*it);
 564                 prevLineCounter = lineCounter;
 565                 lineCounter += pt->getNbDistinctLines();
 566                 if ( lineCounter >= beginIndex )
 567                 {
 568                         uint remainingLines = pageNum - (lineCounter - beginIndex );
 569                         pt->reportPart( beginIndex - prevLineCounter, remainingLines, targetLog ); //
 570                         while ( remainingLines != 0 )
 571                         {
 572                                 ++it;
 573                                 if ( it == _Children.end() )
 574                                 {
 575                                         targetLog->displayRawNL( "[END OF LOG]" );
 576                                         return;
 577                                 }
 578                                 pt = (*it);
 579                                 remainingLines -= pt->reportPart( 0, remainingLines, targetLog );
 580                         }
 581                         return;
 582                 }
 583         }
 584 }