Source/cmCTest.h

   1 /*=========================================================================
   2
   3   Program:   CMake - Cross-Platform Makefile Generator
   4   Module:    $RCSfile: cmCTest.h,v $
   5   Language:  C++
   6   Date:      $Date: 2008-09-22 18:04:12 $
   7   Version:   $Revision: 1.102 $
   8
   9   Copyright (c) 2002 Kitware, Inc., Insight Consortium.  All rights reserved.
  10   See Copyright.txt or http://www.cmake.org/HTML/Copyright.html for details.
  11
  12      This software is distributed WITHOUT ANY WARRANTY; without even
  13      the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
  14      PURPOSE.  See the above copyright notices for more information.
  15
  16 =========================================================================*/
  17
  18 #ifndef cmCTest_h
  19 #define cmCTest_h
  20
  21 #include "cmStandardIncludes.h"
  22 #include "cmListFileCache.h"
  23 #include <time.h>
  24
  25 class cmake;
  26 class cmMakefile;
  27 class cmCTestGenericHandler;
  28 class cmGeneratedFileStream;
  29 class cmCTestCommand;
  30
  31 #define cmCTestLog(ctSelf, logType, msg) \
  32   do { \
  33   cmOStringStream cmCTestLog_msg; \
  34   cmCTestLog_msg << msg; \
  35   (ctSelf)->Log(cmCTest::logType, __FILE__, __LINE__,\
  36                 cmCTestLog_msg.str().c_str());\
  37   } while ( 0 )
  38
  39 #ifdef cerr
  40 #  undef cerr
  41 #endif
  42 #define cerr no_cerr_use_cmCTestLog
  43
  44 #ifdef cout
  45 #  undef cout
  46 #endif
  47 #define cout no_cout_use_cmCTestLog
  48
  49 class cmCTest
  50 {
  51 public:
  52   typedef std::vector<cmStdString> VectorOfStrings;
  53   typedef std::set<cmStdString> SetOfStrings;
  54
  55   ///! Process Command line arguments
  56   int Run(std::vector<std::string> &, std::string* output = 0);
  57
  58   /**
  59    * Initialize and finalize testing
  60    */
  61   int Initialize(const char* binary_dir, bool new_tag = false,
  62     bool verbose_tag = true);
  63   bool InitializeFromCommand(cmCTestCommand* command, bool first = false);
  64   void Finalize();
  65
  66   /**
  67    * Process the tests. This is the main routine. The execution of the
  68    * tests should look like this:
  69    *
  70    * ctest foo;
  71    * foo.Initialize();
  72    * // Set some things on foo
  73    * foo.ProcessTests();
  74    * foo.Finalize();
  75    */
  76   int ProcessTests();
  77
  78   /*
  79    * A utility function that returns the nightly time
  80    */
  81   struct tm* GetNightlyTime(std::string str,
  82     bool tomorrowtag);
  83
  84   /*
  85    * Is the tomorrow tag set?
  86    */
  87   bool GetTomorrowTag() { return this->TomorrowTag; };
  88
  89   /**
  90    * Try to run tests of the project
  91    */
  92   int TestDirectory(bool memcheck);
  93
  94   ///! what is the configuraiton type, e.g. Debug, Release etc.
  95   std::string const& GetConfigType();
  96   double GetTimeOut() { return this->TimeOut; }
  97   void SetTimeOut(double t) { this->TimeOut = t; }
  98   // how many test to run at the same time
  99   int GetParallelLevel() { return this->ParallelLevel; }
 100   void SetParallelLevel(int t) { this->ParallelLevel = t; }
 101
 102   bool GetParallelSubprocess() { return this->ParallelSubprocess; }
 103   void SetParallelSubprocess() { this->ParallelSubprocess = true; }
 104
 105   void SetParallelSubprocessId(int id) { this->ParallelSubprocessId = id;}
 106   int GetParallelSubprocessId() { return this->ParallelSubprocessId;}
 107   const char* GetParallelCacheFile()
 108     { return this->ParallelCacheFile.c_str();}
 109   void SetParallelCacheFile(const char* c) { this->ParallelCacheFile = c; }
 110
 111   /**
 112    * Check if CTest file exists
 113    */
 114   bool CTestFileExists(const std::string& filename);
 115   bool AddIfExists(SetOfStrings& files, const char* file);
 116
 117   /**
 118    * Set the cmake test
 119    */
 120   bool SetTest(const char*, bool report = true);
 121
 122   /**
 123    * Set the cmake test mode (experimental, nightly, continuous).
 124    */
 125   void SetTestModel(int mode);
 126   int GetTestModel() { return this->TestModel; };
 127
 128   std::string GetTestModelString();
 129   static int GetTestModelFromString(const char* str);
 130   static std::string CleanString(const std::string& str);
 131   std::string GetCTestConfiguration(const char *name);
 132   void SetCTestConfiguration(const char *name, const char* value);
 133   void EmptyCTestConfiguration();
 134
 135   /**
 136    * constructor and destructor
 137    */
 138   cmCTest();
 139   ~cmCTest();
 140
 141   //! Set the notes files to be created.
 142   void SetNotesFiles(const char* notes);
 143
 144   void PopulateCustomVector(cmMakefile* mf, const char* definition,
 145     VectorOfStrings& vec);
 146   void PopulateCustomInteger(cmMakefile* mf, const char* def,
 147     int& val);
 148
 149   ///! Get the current time as string
 150   std::string CurrentTime();
 151
 152   /**
 153    * Return the time remaianing that the script is allowed to run in
 154    * seconds if the user has set the variable CTEST_TIME_LIMIT. If that has
 155    * not been set it returns 1e7 seconds
 156    */
 157   double GetRemainingTimeAllowed();
 158
 159   ///! Open file in the output directory and set the stream
 160   bool OpenOutputFile(const std::string& path,
 161                       const std::string& name,
 162                       cmGeneratedFileStream& stream,
 163                       bool compress = false);
 164
 165   ///! Convert string to something that is XML safe
 166   static std::string MakeXMLSafe(const std::string&);
 167
 168   ///! Should we only show what we would do?
 169   bool GetShowOnly();
 170
 171
 172   ///! The max output width
 173   int GetMaxTestNameWidth() const;
 174
 175   /**
 176    * Run a single executable command and put the stdout and stderr
 177    * in output.
 178    *
 179    * If verbose is false, no user-viewable output from the program
 180    * being run will be generated.
 181    *
 182    * If timeout is specified, the command will be terminated after
 183    * timeout expires. Timeout is specified in seconds.
 184    *
 185    * Argument retVal should be a pointer to the location where the
 186    * exit code will be stored. If the retVal is not specified and
 187    * the program exits with a code other than 0, then the this
 188    * function will return false.
 189    *
 190    * If the command has spaces in the path the caller MUST call
 191    * cmSystemTools::ConvertToRunCommandPath on the command before passing
 192    * it into this function or it will not work.  The command must be correctly
 193    * escaped for this to with spaces.
 194    */
 195   bool RunCommand(const char* command,
 196     std::string* stdOut, std::string* stdErr,
 197     int* retVal = 0, const char* dir = 0, double timeout = 0.0);
 198
 199   //! Start CTest XML output file
 200   void StartXML(std::ostream& ostr);
 201
 202   //! End CTest XML output file
 203   void EndXML(std::ostream& ostr);
 204
 205   //! Run command specialized for make and configure. Returns process status
 206   // and retVal is return value or exception.
 207   int RunMakeCommand(const char* command, std::string* output,
 208     int* retVal, const char* dir, int timeout,
 209     std::ofstream& ofs);
 210
 211   /*
 212    * return the current tag
 213    */
 214   std::string GetCurrentTag();
 215
 216   //! Get the path to the build tree
 217   std::string GetBinaryDir();
 218
 219   //! Get the short path to the file. This means if the file is in binary or
 220   //source directory, it will become /.../relative/path/to/file
 221   std::string GetShortPathToFile(const char* fname);
 222
 223   //! Get the path to CTest
 224   const char* GetCTestExecutable() { return this->CTestSelf.c_str(); }
 225   const char* GetCMakeExecutable() { return this->CMakeSelf.c_str(); }
 226
 227   enum {
 228     EXPERIMENTAL,
 229     NIGHTLY,
 230     CONTINUOUS
 231   };
 232
 233   // provide some more detailed info on the return code for ctest
 234   enum {
 235     UPDATE_ERRORS    = 0x01,
 236     CONFIGURE_ERRORS = 0x02,
 237     BUILD_ERRORS     = 0x04,
 238     TEST_ERRORS      = 0x08,
 239     MEMORY_ERRORS    = 0x10,
 240     COVERAGE_ERRORS  = 0x20,
 241     SUBMIT_ERRORS    = 0x40
 242   };
 243
 244   ///! Are we producing XML
 245   bool GetProduceXML();
 246   void SetProduceXML(bool v);
 247
 248   //! Run command specialized for tests. Returns process status and retVal is
 249   // return value or exception.
 250   int RunTest(std::vector<const char*> args, std::string* output, int *retVal,
 251     std::ostream* logfile, double testTimeOut);
 252
 253   /**
 254    * Execute handler and return its result. If the handler fails, it returns
 255    * negative value.
 256    */
 257   int ExecuteHandler(const char* handler);
 258
 259   /*
 260    * Get the handler object
 261    */
 262   cmCTestGenericHandler* GetHandler(const char* handler);
 263   cmCTestGenericHandler* GetInitializedHandler(const char* handler);
 264
 265   /*
 266    * Set the CTest variable from CMake variable
 267    */
 268   bool SetCTestConfigurationFromCMakeVariable(cmMakefile* mf,
 269     const char* dconfig, const char* cmake_var);
 270
 271   //! Make string safe to be send as an URL
 272   static std::string MakeURLSafe(const std::string&);
 273
 274   //! Should ctect configuration be updated. When using new style ctest
 275   // script, this should be true.
 276   void SetSuppressUpdatingCTestConfiguration(bool val)
 277     {
 278     this->SuppressUpdatingCTestConfiguration = val;
 279     }
 280
 281   //! Add overwrite to ctest configuration.
 282   // The format is key=value
 283   void AddCTestConfigurationOverwrite(const char* encstr);
 284
 285   //! Create XML file that contains all the notes specified
 286   int GenerateNotesFile(const std::vector<cmStdString> &files);
 287
 288   //! Submit extra files to the server
 289   bool SubmitExtraFiles(const char* files);
 290   bool SubmitExtraFiles(const std::vector<cmStdString> &files);
 291
 292   //! Set the output log file name
 293   void SetOutputLogFileName(const char* name);
 294
 295   //! Set the visual studio or Xcode config type
 296   void SetConfigType(const char* ct);
 297
 298   //! Various log types
 299   enum {
 300     DEBUG = 0,
 301     OUTPUT,
 302     HANDLER_OUTPUT,
 303     HANDLER_VERBOSE_OUTPUT,
 304     WARNING,
 305     ERROR_MESSAGE,
 306     OTHER
 307   };
 308
 309   //! Add log to the output
 310   void Log(int logType, const char* file, int line, const char* msg);
 311
 312   //! Get the version of dart server
 313   int GetDartVersion() { return this->DartVersion; }
 314
 315   //! Add file to be submitted
 316   void AddSubmitFile(const char* name);
 317   SetOfStrings* GetSubmitFiles() { return &this->SubmitFiles; }
 318
 319   //! Read the custom configuration files and apply them to the current ctest
 320   int ReadCustomConfigurationFileTree(const char* dir, cmMakefile* mf);
 321
 322   std::vector<cmStdString> &GetInitialCommandLineArguments()
 323   { return this->InitialCommandLineArguments; };
 324
 325   //! Set the track to submit to
 326   void SetSpecificTrack(const char* track);
 327   const char* GetSpecificTrack();
 328
 329   bool GetVerbose() { return this->Verbose;}
 330   bool GetExtraVerbose() { return this->ExtraVerbose;}
 331 private:
 332   std::string ConfigType;
 333   bool Verbose;
 334   bool ExtraVerbose;
 335   bool ProduceXML;
 336
 337   bool ForceNewCTestProcess;
 338
 339   bool RunConfigurationScript;
 340
 341   int GenerateNotesFile(const char* files);
 342
 343   // these are helper classes
 344   typedef std::map<cmStdString,cmCTestGenericHandler*> t_TestingHandlers;
 345   t_TestingHandlers TestingHandlers;
 346
 347   bool ShowOnly;
 348
 349   enum {
 350     FIRST_TEST     = 0,
 351     UPDATE_TEST    = 1,
 352     START_TEST     = 2,
 353     CONFIGURE_TEST = 3,
 354     BUILD_TEST     = 4,
 355     TEST_TEST      = 5,
 356     COVERAGE_TEST  = 6,
 357     MEMCHECK_TEST  = 7,
 358     SUBMIT_TEST    = 8,
 359     NOTES_TEST     = 9,
 360     ALL_TEST       = 10,
 361     LAST_TEST      = 11
 362   };
 363
 364   //! Map of configuration properties
 365   typedef std::map<cmStdString, cmStdString> CTestConfigurationMap;
 366
 367   std::string             CTestConfigFile;
 368   CTestConfigurationMap CTestConfiguration;
 369   CTestConfigurationMap CTestConfigurationOverwrites;
 370   int                     Tests[LAST_TEST];
 371
 372   std::string             CurrentTag;
 373   bool                    TomorrowTag;
 374
 375   int                     TestModel;
 376   std::string             SpecificTrack;
 377
 378   double                  TimeOut;
 379
 380   int                     MaxTestNameWidth;
 381
 382   std::string             ParallelCacheFile;
 383   int                     ParallelLevel;
 384   int                     ParallelSubprocessId;
 385   bool                    ParallelSubprocess;
 386   int                     CompatibilityMode;
 387
 388   // information for the --build-and-test options
 389   std::string              CMakeSelf;
 390   std::string              CTestSelf;
 391   std::string              BinaryDir;
 392
 393   std::string              NotesFiles;
 394
 395
 396   bool                     InteractiveDebugMode;
 397
 398   bool                     ShortDateFormat;
 399
 400   bool                     CompressXMLFiles;
 401
 402   void BlockTestErrorDiagnostics();
 403
 404
 405   //! parse the option after -D and convert it into the appropriate steps
 406   bool AddTestsForDashboardType(std::string &targ);
 407
 408   //! parse and process most common command line arguments
 409   void HandleCommandLineArguments(size_t &i,
 410                                   std::vector<std::string> &args);
 411
 412   //! hande the -S -SP and -SR arguments
 413   void HandleScriptArguments(size_t &i,
 414                              std::vector<std::string> &args,
 415                              bool &SRArgumentSpecified);
 416
 417   //! Reread the configuration file
 418   bool UpdateCTestConfiguration();
 419
 420   //! Create not from files.
 421   int GenerateCTestNotesOutput(std::ostream& os,
 422     const VectorOfStrings& files);
 423
 424   ///! Find the running cmake
 425   void FindRunningCMake();
 426
 427   //! Check if the argument is the one specified
 428   bool CheckArgument(const std::string& arg, const char* varg1,
 429     const char* varg2 = 0);
 430
 431   bool                      SuppressUpdatingCTestConfiguration;
 432
 433   bool Debug;
 434   bool ShowLineNumbers;
 435   bool Quiet;
 436
 437   int  DartVersion;
 438
 439   std::set<cmStdString> SubmitFiles;
 440   std::vector<cmStdString> InitialCommandLineArguments;
 441
 442   int SubmitIndex;
 443
 444   cmGeneratedFileStream* OutputLogFile;
 445   int OutputLogFileLastTag;
 446 };
 447
 448 class cmCTestLogWrite
 449 {
 450 public:
 451   cmCTestLogWrite(const char* data, size_t length)
 452     : Data(data), Length(length) {}
 453
 454   const char* Data;
 455   size_t Length;
 456 };
 457
 458 inline std::ostream& operator<< (std::ostream& os, const cmCTestLogWrite& c)
 459 {
 460   if (!c.Length)
 461     {
 462     return os;
 463     }
 464   os.write(c.Data, c.Length);
 465   os.flush();
 466   return os;
 467 }
 468
 469 #endif