build-scripts/gulpfile-test.js

   1 'use strict';
   2
   3 var gulp = require('gulp');
   4 var gutil = require('gulp-util');
   5 var runSequence = require('run-sequence');
   6 var using = require('gulp-using');
   7 var codecov = require('gulp-codecov');
   8 var lcovMerger = require('lcov-result-merger');
   9 var path = require('path');
  10 var mkdirp = require('mkdirp');
  11 var glob = require('glob');
  12 var childProcessPromise = require('./child-process-promise');
  13 var del = require('del');
  14 var Promise = require('bluebird');
  15
  16 var argv = require('yargs')
  17   .option('test-suite', {
  18     describe: 'test-suite to run',
  19     choices: ['local', 'mocha', 'docker'],
  20     default: 'local'
  21   })
  22   .option('test-fast', {
  23     describe: 'only run fast tests',
  24     boolean: true,
  25     default: false
  26   })
  27   .option('test-parallel', {
  28     describe: 'run tests in parallel',
  29     boolean: true,
  30     default: false
  31   })
  32   .option('test-coverage', {
  33     describe: 'generate coverage on tests',
  34     boolean: true,
  35     default: false
  36   })
  37   .option('test-bail', {
  38     describe: 'bail on first failure',
  39     boolean: true,
  40     default: false
  41   })
  42   .option('test-xunit-reports', {
  43     describe: 'generate xunit reports',
  44     boolean: true,
  45     default: false
  46   })
  47   .option('test-grep', {
  48     alias: 'g',
  49     describe: 'grep tests',
  50     type: 'string'
  51   })
  52   .option('test-critical-only', {
  53     describe: 'only test critical tests',
  54     boolean: true,
  55     default: false
  56   })
  57   .help('test-help').argv;
  58
  59 /* Don't do clean in gulp, use make */
  60 var RUN_TESTS_IN_PARALLEL = argv['test-parallel'];
  61 var generateCoverage = argv['test-coverage'];
  62 var generateXUnitReports = argv['test-xunit-reports'];
  63 var bail = argv['test-bail'];
  64 var testSuite = argv['test-suite'];
  65 var fast = argv['test-fast'];
  66 var grep = argv['test-grep'];
  67 var criticalOnly = argv['test-critical-only'];
  68 const disableGitHubTests =
  69   process.env.DISABLE_GITHUB_TESTS && JSON.parse(process.env.DISABLE_GITHUB_TESTS);
  70
  71 var testModules = {};
  72
  73 var modulesWithTest = glob.sync('./modules/*/test');
  74
  75 modulesWithTest.forEach(function(testDir) {
  76   var moduleDir = path.dirname(testDir);
  77   var moduleName = path.basename(moduleDir);
  78   testModules[moduleName] = {
  79     files: [path.join('modules', moduleName, 'test')],
  80     isCritical: false
  81   };
  82 });
  83
  84 testModules['request-tests'] = {
  85   files: [
  86     './test/request-api-tests/'
  87     // XXX: Skipped because Gitter login is now disabled (#gitter-sunset)
  88     //'./test/request-web-tests/'
  89   ],
  90   options: {
  91     // These tests load the entire app, so mocha will sometimes timeout before it even runs the tests
  92     timeout: 30000
  93   },
  94   isCritical: false
  95 };
  96
  97 testModules.integration = {
  98   files: ['./test/integration/'],
  99   isCritical: true
 100 };
 101
 102 // eslint-disable-next-line max-statements, complexity
 103 function spawnMochaProcess(moduleName, options, files) {
 104   var executable;
 105   var args;
 106
 107   var argReporter = 'spec';
 108   var argTimeout = (options && options.timeout) || 10000;
 109   var argBail = bail;
 110
 111   var env = {
 112     SKIP_BADGER_TESTS: 1,
 113     BLUEBIRD_DEBUG: 1,
 114     NO_AUTO_INDEX: 1,
 115     TZ: 'UTC',
 116     GITTER_TEST: 1
 117   };
 118
 119   if (generateCoverage) {
 120     executable = './node_modules/.bin/nyc';
 121     args = [
 122       '--cache',
 123       '--report-dir',
 124       'output/coverage-reports/' + moduleName,
 125       '--reporter',
 126       'lcov',
 127       './node_modules/.bin/mocha'
 128     ];
 129
 130     mkdirp.sync('output/coverage-reports/' + moduleName);
 131   } else {
 132     executable = './node_modules/.bin/mocha';
 133     args = [];
 134   }
 135
 136   // This is needed because Mocha no longer force exits, https://boneskull.com/mocha-v4-nears-release/#mochawontforceexit
 137   args.push('--exit');
 138
 139   if (testSuite === 'docker') {
 140     env.HOME = '/tmp'; // Needs to be writeable inside docker for `nyc`
 141     env.NODE_ENV = 'test-docker';
 142   }
 143
 144   var argGrep = grep;
 145   var argInvert = false;
 146   if (fast) {
 147     argGrep = '#slow';
 148     argInvert = true;
 149     argBail = true;
 150   } else if (disableGitHubTests) {
 151     argGrep = '#github';
 152     argInvert = true;
 153   }
 154
 155   if (generateXUnitReports) {
 156     mkdirp.sync('output/test-reports/');
 157
 158     argReporter = 'mocha-multi';
 159     env.multi = 'spec=- xunit=output/test-reports/' + moduleName + '.xml';
 160     gutil.log('Will write xunit reports to', 'output/test-reports/' + moduleName + '.xml');
 161   }
 162
 163   args.push('--recursive');
 164
 165   args.push('-R');
 166   args.push(argReporter);
 167
 168   if (argGrep) {
 169     args.push('--grep');
 170     args.push(argGrep);
 171   }
 172
 173   if (argBail) {
 174     args.push('--bail');
 175   }
 176
 177   if (argInvert) {
 178     args.push('--invert');
 179   }
 180
 181   args.push('--timeout');
 182   args.push(argTimeout);
 183
 184   args = args.concat(files);
 185   gutil.log('Running tests with', executable, args.join(' '));
 186
 187   return childProcessPromise.spawn(executable, args, env);
 188 }
 189
 190 var subTasks = [];
 191 var testingErrors = [];
 192
 193 Object.keys(testModules).forEach(function(moduleName) {
 194   var definition = testModules[moduleName];
 195
 196   if (criticalOnly && !definition.isCritical) {
 197     return;
 198   }
 199
 200   var testTaskName = 'test:test:' + moduleName;
 201   subTasks.push(testTaskName);
 202
 203   gulp.task(testTaskName, function() {
 204     return spawnMochaProcess(moduleName, definition.options, definition.files).catch(function(err) {
 205       if (bail) {
 206         throw new Error('Test module ' + moduleName + ' failed. ' + err);
 207       }
 208
 209       testingErrors.push(err);
 210     });
 211   });
 212 });
 213
 214 subTasks.push('test:test:jest');
 215 gulp.task('test:test:jest', function() {
 216   return childProcessPromise.spawn('npm', ['run', 'jest']);
 217 });
 218
 219 gulp.task('test:pre-test', function() {
 220   var env = {};
 221
 222   if (testSuite === 'docker') {
 223     env.NODE_ENV = 'test-docker';
 224   }
 225
 226   return childProcessPromise.spawn('./scripts/utils/ensure-mongodb-indexes.js', [], env);
 227 });
 228
 229 if (RUN_TESTS_IN_PARALLEL) {
 230   // Run tests in parallel
 231   gulp.task('test:test', subTasks, function() {
 232     if (testingErrors.length) {
 233       return Promise.reject(testingErrors[0]);
 234     }
 235   });
 236 } else {
 237   // Run tests in sequence
 238   gulp.task('test:test', function(callback) {
 239     gutil.log('Run sequence for test:test', subTasks.join(','));
 240
 241     var args = subTasks.concat(function(err) {
 242       if (err) return callback(err);
 243
 244       if (testingErrors.length) {
 245         return callback(testingErrors[0]);
 246       }
 247
 248       return callback();
 249     });
 250
 251     runSequence.apply(null, args);
 252   });
 253 }
 254
 255 /**
 256  * Hook into post test
 257  */
 258 if (generateCoverage) {
 259   gulp.task('test:post-test', ['test:post-test:merge-lcov', 'test:post-test:submit-codecov']);
 260 }
 261
 262 gulp.task('test:post-test:merge-lcov', function() {
 263   return gulp
 264     .src('output/coverage-reports/**/lcov.info')
 265     .pipe(using())
 266     .pipe(lcovMerger())
 267     .pipe(gulp.dest('output/coverage-reports/merged/'));
 268 });
 269
 270 gulp.task('test:post-test:submit-codecov', ['test:post-test:merge-lcov'], function() {
 271   process.env.CODECOV_TOKEN = '4d30a5c7-3839-4396-a2fd-d8f9a68a5c3a';
 272
 273   return gulp
 274     .src('output/coverage-reports/merged/lcov.info')
 275     .pipe(
 276       codecov({
 277         disable: 'gcov'
 278       })
 279     )
 280     .on('error', function(err) {
 281       gutil.log(err);
 282       this.emit('end');
 283     });
 284 });
 285
 286 /**
 287  * Hook into the clean stage
 288  */
 289 gulp.task('test:clean', function(cb) {
 290   del(['.nyc_output/'], cb);
 291 });