tests/glean/main.cpp

   1 // BEGIN_COPYRIGHT -*- glean -*-
   2 //
   3 // Copyright (C) 1999  Allen Akin   All Rights Reserved.
   4 //
   5 // Permission is hereby granted, free of charge, to any person
   6 // obtaining a copy of this software and associated documentation
   7 // files (the "Software"), to deal in the Software without
   8 // restriction, including without limitation the rights to use,
   9 // copy, modify, merge, publish, distribute, sublicense, and/or
  10 // sell copies of the Software, and to permit persons to whom the
  11 // Software is furnished to do so, subject to the following
  12 // conditions:
  13 //
  14 // The above copyright notice and this permission notice shall be
  15 // included in all copies or substantial portions of the
  16 // Software.
  17 //
  18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  19 // KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  20 // WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  21 // PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL ALLEN AKIN BE
  22 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
  23 // AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
  24 // OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  25 // DEALINGS IN THE SOFTWARE.
  26 //
  27 // END_COPYRIGHT
  28
  29
  30
  31
  32 // main.cpp:  main program for Glean
  33
  34 #include <cassert>
  35 #include <cstring>
  36 #include <iostream>
  37 #include <string>
  38 #include <vector>
  39 #include <cstring>
  40 #include <algorithm>
  41 #include "options.h"
  42 #include "environ.h"
  43 #include "test.h"
  44 #include "version.h"
  45 #include "lex.h"
  46 #include "dsfilt.h"
  47
  48 using namespace std;
  49
  50 using namespace GLEAN;
  51
  52 char* mandatoryArg(int argc, char* argv[], int i);
  53 void selectTests(Options& o, vector<string>& allTestNames, int argc,
  54         char* argv[], int i);
  55 void usage(char* command);
  56 void listTests(const Test *tests, bool verbose);
  57
  58 int
  59 main(int argc, char* argv[]) {
  60
  61         // Until someone gets around to writing a fancy GUI front-end,
  62         // we'll set options the old-fashioned way.
  63         Options o;
  64         bool visFilter = false;
  65
  66         vector<string> allTestNames;
  67         for (Test* t = Test::testList; t; t = t->nextTest)
  68                 allTestNames.push_back(t->name);
  69         sort(allTestNames.begin(), allTestNames.end());
  70         o.selectedTests = allTestNames;
  71
  72         for (int i = 1; i < argc; ++i) {
  73                 if (!strcmp(argv[i], "--help")) {
  74                         usage(argv[0]);
  75                 } else if (!strcmp(argv[i], "-v")
  76                     || !strcmp(argv[i], "--verbose")) {
  77                         ++o.verbosity;
  78                 } else if (!strcmp(argv[i], "-r")
  79                     || !strcmp(argv[i], "--run")) {
  80                         o.mode = Options::run;
  81                         ++i;
  82                         o.db1Name = mandatoryArg(argc, argv, i);
  83                 } else if (!strcmp(argv[i], "-o")
  84                     || !strcmp(argv[i], "--overwrite")) {
  85                         o.overwrite = true;
  86                 } else if (!strcmp(argv[i], "--quick")) {
  87                         o.quick = true;
  88                 } else if (!strcmp(argv[i], "-c")
  89                     || !strcmp(argv[i], "--compare")) {
  90                         o.mode = Options::compare;
  91                         ++i;
  92                         o.db1Name = mandatoryArg(argc, argv, i);
  93                         ++i;
  94                         o.db2Name = mandatoryArg(argc, argv, i);
  95                 } else if (!strcmp(argv[i], "--visuals")) {
  96                         visFilter = true;
  97                         ++i;
  98                         o.visFilter = mandatoryArg(argc, argv, i);
  99                 } else if (!strcmp(argv[i], "-t")
 100                            || !strcmp(argv[i], "--tests")) {
 101                         ++i;
 102                         selectTests(o, allTestNames, argc, argv, i);
 103                 } else if (!strcmp(argv[i], "--listtests")) {
 104                         o.mode = Options::listtests;
 105 #           if defined(__X11__)
 106                 } else if (!strcmp(argv[i], "-display")
 107                     || !strcmp(argv[i], "--display")) {
 108                         ++i;
 109                         o.dpyName = mandatoryArg(argc, argv, i);
 110 #           endif
 111                 } else {
 112                         usage(argv[0]);
 113                 }
 114         }
 115
 116         if (o.mode == Options::notSet)
 117                 usage(argv[0]);
 118
 119         if (o.mode == Options::listtests) {
 120                 listTests(Test::testList, o.verbosity);
 121                 exit(0);
 122         }
 123
 124         if (o.quick && !visFilter) {
 125                 // If we have --quick but not --visuals then limit testing to
 126                 // a single RGB, Z, Stencil visual.
 127                 o.visFilter = "rgb && z>0 && s>0 && conformant";
 128                 o.maxVisuals = 1;
 129         }
 130
 131         // Create the test environment, then invoke each test to generate
 132         // results or compare two previous runs.
 133         try {
 134                 Environment e(o);
 135                 switch (o.mode) {
 136                 case Options::run:
 137                 {
 138                         for (Test* t = Test::testList; t; t = t->nextTest)
 139                                 if (binary_search(o.selectedTests.begin(),
 140                                     o.selectedTests.end(), t->name))
 141                                         t->run(e);
 142                         break;
 143                 }
 144                 case Options::compare:
 145                 {
 146                         for (Test* t = Test::testList; t; t = t->nextTest)
 147                                 if (binary_search(o.selectedTests.begin(),
 148                                     o.selectedTests.end(), t->name))
 149                                         try {
 150                                                 t->compare(e);
 151                                         }
 152                                         catch (Test::CantOpenResultsFile e) {
 153                                                 // For comparisons, we want to
 154                                                 // continue running even if a
 155                                                 // test result file can't be
 156                                                 // opened.  We report the
 157                                                 // problem here, but don't exit
 158                                                 // as we would in the catch{}
 159                                                 // below.
 160                                                 cerr << "Can't open results file for test "
 161                                                         << e.testName
 162                                                         << " in database "
 163                                                         << e.dbName
 164                                                         << '\n';
 165                                         }
 166                         break;
 167                 }
 168                 default:
 169                         cerr << "Bad run mode in main()\n";
 170                         break;
 171                 }
 172         }
 173 #if defined(__X11__)
 174         catch (WindowSystem::CantOpenDisplay) {
 175                 cerr << "can't open display " << o.dpyName << "\n";
 176                 exit(1);
 177         }
 178 #endif
 179         catch (WindowSystem::NoOpenGL) {
 180                 cerr << "display doesn't support OpenGL\n";
 181                 exit(1);
 182         }
 183         catch (DrawingSurfaceFilter::Syntax e) {
 184                 cerr << "Syntax error in visual selection criteria:\n"
 185                         "'" << o.visFilter << "'\n";
 186                 for (int i = 0; i < e.position; ++i)
 187                         cerr << ' ';
 188                 cerr << "^ " << e.err << '\n';
 189                 exit(1);
 190         }
 191         catch (Environment::DBExists) {
 192                 cerr << "Won't overwrite existing database " << o.db1Name
 193                         << "\n";
 194                 exit(1);
 195         }
 196         catch (Environment::DBCantOpen e) {
 197                 cerr << "Can't open database directory " << *e.db << "\n";
 198                 exit(1);
 199         }
 200         catch (Test::CantOpenResultsFile e) {
 201                 cerr << "Can't open results file for test " << e.testName
 202                         << " in database " << e.dbName << '\n';
 203                 exit(1);
 204         }
 205         catch (...) {
 206                 cerr << "caught an unexpected error in main()\n";
 207                 exit(1);
 208         }
 209
 210         return 0;
 211 } // main
 212
 213
 214 char*
 215 mandatoryArg(int argc, char* argv[], int i) {
 216         if (i < argc && argv[i][0] != '-')
 217                 return argv[i];
 218         usage(argv[0]);
 219         /*NOTREACHED*/
 220         return 0;
 221 } // mandatoryArg
 222
 223
 224 void
 225 selectTests(Options& o, vector<string>& allTestNames, int argc, char* argv[],
 226     int i) {
 227         if (i >= argc)
 228                 usage(argv[0]);
 229
 230         // At present, we deal with the following syntax:
 231         //      [+] testname {(+|-) testname}
 232         //              Assume we're running none of the tests, then include
 233         //              those preceded by "+" and exclude those preceded by
 234         //              "-".
 235         //      - testname {(+|-) testname}
 236         //              Assume we're running all of the tests, then exclude
 237         //              those preceded by "-" and include those preceded by
 238         //              "+".
 239         // XXX It would be nice to support the syntax "@filename" to mean
 240         // "the list of tests given in the named file."  This could be
 241         // preceded by "+" or "-" just like an ordinary test name, or maybe
 242         // the +/- should be required in the file itself.
 243
 244         struct SyntaxError {
 245                 int position;
 246                 SyntaxError(int pos): position(pos) { }
 247                 };
 248
 249         Lex lex(argv[i]);
 250         try {
 251                 lex.next();
 252                 if (lex.token == Lex::MINUS)
 253                         o.selectedTests = allTestNames;
 254                 else
 255                         o.selectedTests.resize(0);
 256
 257                 while (lex.token != Lex::END) {
 258                         bool inserting = true;
 259                         if (lex.token == Lex::MINUS) {
 260                                 inserting = false;
 261                                 lex.next();
 262                         } else if (lex.token == Lex::PLUS)
 263                                 lex.next();
 264
 265                         if (lex.token != Lex::ID)
 266                                 throw SyntaxError(lex.position());
 267
 268                         if (!binary_search(allTestNames.begin(),
 269                             allTestNames.end(), lex.id))
 270                                 cerr << "Warning: " << lex.id << " ignored;"
 271                                         " not a valid test name.\n";
 272                         else {
 273                                 vector<string>::iterator p =
 274                                     lower_bound(o.selectedTests.begin(),
 275                                         o.selectedTests.end(), lex.id);
 276                                 if (inserting) {
 277                                         if (p == o.selectedTests.end()
 278                                           || *p != lex.id)
 279                                                 o.selectedTests.insert(p,
 280                                                     lex.id);
 281                                 } else {
 282                                         // removing
 283                                         if (p != o.selectedTests.end()
 284                                           && *p == lex.id)
 285                                                 o.selectedTests.erase(p);
 286                                 }
 287                         }
 288                         lex.next();
 289                 }
 290         }
 291         catch (Lex::Lexical e) {
 292                 cerr << "Lexical error in test inclusion/exclusion list:\n"
 293                         "'" << argv[i] << "'\n";
 294                 for (int i = 0; i < e.position; ++i)
 295                         cerr << ' ';
 296                 cerr << "^ " << e.err << "\n\n";
 297                 usage(argv[0]);
 298         }
 299         catch (SyntaxError e) {
 300                 cerr << "'" << argv[i] << "'\n";
 301                 for (int i = 0; i < e.position; ++i)
 302                         cerr << ' ';
 303                 cerr << "^ Syntax error in test inclusion/exclusion list\n\n";
 304                 usage(argv[0]);
 305         }
 306 } // selectTests
 307
 308
 309 void
 310 listTests(const Test *tests, bool verbose) {
 311         for (const Test *t = tests; t; t = t->nextTest) {
 312                 cout << t->name << (verbose? ":" : "") << "\n";
 313                 if (verbose) {
 314                         cout << t->description;
 315                         cout << '\n';
 316                 }
 317         }
 318 }
 319
 320
 321 void
 322 usage(char* command) {
 323         cerr << GLEAN::versionString << '\n';
 324         cerr << "Usage:  " << command << " mode [options]\n"
 325 "\n"
 326 "mode:\n"
 327 "       (-r|--run) results-directory\n"
 328 "   or  (-c|--compare) old-results-dir new-results-dir\n"
 329 "\n"
 330 "options:\n"
 331 "       (-v|--verbose)             # each occurrence increases\n"
 332 "                                  # verbosity of output\n"
 333 "       (-o|--overwrite)           # overwrite existing results database\n"
 334 "       --visuals 'filter-string'  # select subset of visuals (FBConfigs,\n"
 335 "                                  # pixel formats) to test\n"
 336 "       (-t|--tests) {(+|-)test}   # choose tests to include (+) or exclude (-)\n"
 337 "       --quick                    # run fewer tests to reduce test time\n"
 338 "       --listtests                # list test names and exit\n"
 339 "       --help                     # display usage information\n"
 340 #if defined(__X11__)
 341 "       -display X11-display-name  # select X11 display to use\n"
 342 "           (or --display)\n"
 343 #elif defined(__WIN__)
 344 #endif
 345                 ;
 346         exit(1);
 347 } // usage