test/runner/run.js

   1 import chalk from "chalk";
   2 import { asyncExitHook, gracefulExit } from "exit-hook";
   3 import { getLatestBrowser } from "./browserstack/api.js";
   4 import { buildBrowserFromString } from "./browserstack/buildBrowserFromString.js";
   5 import { localTunnel } from "./browserstack/local.js";
   6 import { reportEnd, reportTest } from "./reporter.js";
   7 import { createTestServer } from "./createTestServer.js";
   8 import { buildTestUrl } from "./lib/buildTestUrl.js";
   9 import { generateHash, printModuleHashes } from "./lib/generateHash.js";
  10 import { getBrowserString } from "./lib/getBrowserString.js";
  11 import { modules as allModules } from "./flags/modules.js";
  12 import { cleanupAllBrowsers, touchBrowser } from "./browsers.js";
  13 import {
  14         addRun,
  15         getNextBrowserTest,
  16         hardRetryTest,
  17         retryTest,
  18         runAll
  19 } from "./queue.js";
  20
  21 const EXIT_HOOK_WAIT_TIMEOUT = 60 * 1000;
  22
  23 /**
  24  * Run modules in parallel in different browser instances.
  25  */
  26 export async function run( {
  27         browser: browserNames = [],
  28         browserstack,
  29         concurrency,
  30         debug,
  31         esm,
  32         hardRetries,
  33         headless,
  34         isolate,
  35         module: modules = [],
  36         retries = 0,
  37         runId,
  38         verbose
  39 } ) {
  40         if ( !browserNames.length ) {
  41                 browserNames = [ "chrome" ];
  42         }
  43         if ( !modules.length ) {
  44                 modules = allModules;
  45         }
  46         if ( headless && debug ) {
  47                 throw new Error(
  48                         "Cannot run in headless mode and debug mode at the same time."
  49                 );
  50         }
  51
  52         if ( verbose ) {
  53                 console.log( browserstack ? "Running in BrowserStack." : "Running locally." );
  54         }
  55
  56         const errorMessages = [];
  57         const pendingErrors = {};
  58
  59         // Convert browser names to browser objects
  60         let browsers = browserNames.map( ( b ) => ( { browser: b } ) );
  61         const tunnelId = generateHash(
  62                 `${ Date.now() }-${ modules.join( ":" ) }-${ ( browserstack || [] )
  63                         .concat( browserNames )
  64                         .join( ":" ) }`
  65         );
  66
  67         // A unique identifier for this run
  68         if ( !runId ) {
  69                 runId = tunnelId;
  70         }
  71
  72         // Create the test app and
  73         // hook it up to the reporter
  74         const reports = Object.create( null );
  75         const app = await createTestServer( async( message ) => {
  76                 switch ( message.type ) {
  77                         case "testEnd": {
  78                                 const reportId = message.id;
  79                                 const report = reports[ reportId ];
  80                                 touchBrowser( report.browser );
  81                                 const errors = reportTest( message.data, reportId, report );
  82                                 pendingErrors[ reportId ] ??= Object.create( null );
  83                                 if ( errors ) {
  84                                         pendingErrors[ reportId ][ message.data.name ] = errors;
  85                                 } else {
  86                                         const existing = pendingErrors[ reportId ][ message.data.name ];
  87
  88                                         // Show a message for flakey tests
  89                                         if ( existing ) {
  90                                                 console.log();
  91                                                 console.warn(
  92                                                         chalk.italic(
  93                                                                 chalk.gray( existing.replace( "Test failed", "Test flakey" ) )
  94                                                         )
  95                                                 );
  96                                                 console.log();
  97                                                 delete pendingErrors[ reportId ][ message.data.name ];
  98                                         }
  99                                 }
 100                                 break;
 101                         }
 102                         case "runEnd": {
 103                                 const reportId = message.id;
 104                                 const report = reports[ reportId ];
 105                                 touchBrowser( report.browser );
 106                                 const { failed, total } = reportEnd(
 107                                         message.data,
 108                                         message.id,
 109                                         reports[ reportId ]
 110                                 );
 111                                 report.total = total;
 112
 113                                 // Handle failure
 114                                 if ( failed ) {
 115                                         const retry = retryTest( reportId, retries );
 116
 117                                         // Retry if retryTest returns a test
 118                                         if ( retry ) {
 119                                                 return retry;
 120                                         }
 121
 122                                         // Return early if hardRetryTest returns true
 123                                         if ( await hardRetryTest( reportId, hardRetries ) ) {
 124                                                 return;
 125                                         }
 126                                         errorMessages.push( ...Object.values( pendingErrors[ reportId ] ) );
 127                                 }
 128
 129                                 // Run the next test
 130                                 return getNextBrowserTest( reportId );
 131                         }
 132                         case "ack": {
 133                                 const report = reports[ message.id ];
 134                                 touchBrowser( report.browser );
 135                                 break;
 136                         }
 137                         default:
 138                                 console.warn( "Received unknown message type:", message.type );
 139                 }
 140
 141         // Hide test server request logs in CLI output
 142         }, { quiet: true } );
 143
 144         // Start up local test server
 145         let server;
 146         let port;
 147         await new Promise( ( resolve ) => {
 148
 149                 // Pass 0 to choose a random, unused port
 150                 server = app.listen( 0, () => {
 151                         port = server.address().port;
 152                         resolve();
 153                 } );
 154         } );
 155
 156         if ( !server || !port ) {
 157                 throw new Error( "Server not started." );
 158         }
 159
 160         if ( verbose ) {
 161                 console.log( `Server started on port ${ port }.` );
 162         }
 163
 164         function stopServer() {
 165                 return new Promise( ( resolve ) => {
 166                         server.close( () => {
 167                                 if ( verbose ) {
 168                                         console.log( "Server stopped." );
 169                                 }
 170                                 resolve();
 171                         } );
 172                 } );
 173         }
 174
 175         async function cleanup() {
 176                 console.log( "Cleaning up..." );
 177
 178                 await cleanupAllBrowsers( { verbose } );
 179
 180                 if ( tunnel ) {
 181                         await tunnel.stop();
 182                         if ( verbose ) {
 183                                 console.log( "Stopped BrowserStackLocal." );
 184                         }
 185                 }
 186         }
 187
 188         asyncExitHook(
 189                 async() => {
 190                         await cleanup();
 191                         await stopServer();
 192                 },
 193                 { wait: EXIT_HOOK_WAIT_TIMEOUT }
 194         );
 195
 196         // Start up BrowserStackLocal
 197         let tunnel;
 198         if ( browserstack ) {
 199                 if ( headless ) {
 200                         console.warn(
 201                                 chalk.italic(
 202                                         "BrowserStack does not support headless mode. Running in normal mode."
 203                                 )
 204                         );
 205                         headless = false;
 206                 }
 207
 208                 // Convert browserstack to browser objects.
 209                 // If browserstack is an empty array, fall back
 210                 // to the browsers array.
 211                 if ( browserstack.length ) {
 212                         browsers = browserstack.map( ( b ) => {
 213                                 if ( !b ) {
 214                                         return browsers[ 0 ];
 215                                 }
 216                                 return buildBrowserFromString( b );
 217                         } );
 218                 }
 219
 220                 // Fill out browser defaults
 221                 browsers = await Promise.all(
 222                         browsers.map( async( browser ) => {
 223
 224                                 // Avoid undici connect timeout errors
 225                                 await new Promise( ( resolve ) => setTimeout( resolve, 100 ) );
 226
 227                                 const latestMatch = await getLatestBrowser( browser );
 228                                 if ( !latestMatch ) {
 229                                         console.error(
 230                                                 chalk.red( `Browser not found: ${ getBrowserString( browser ) }.` )
 231                                         );
 232                                         gracefulExit( 1 );
 233                                 }
 234                                 return latestMatch;
 235                         } )
 236                 );
 237
 238                 tunnel = await localTunnel( tunnelId );
 239                 if ( verbose ) {
 240                         console.log( "Started BrowserStackLocal." );
 241
 242                         printModuleHashes( modules );
 243                 }
 244         }
 245
 246         function queueRun( modules, browser ) {
 247                 const fullBrowser = getBrowserString( browser, headless );
 248                 const reportId = generateHash( `${ modules.join( ":" ) } ${ fullBrowser }` );
 249                 reports[ reportId ] = { browser, headless, modules };
 250
 251                 const url = buildTestUrl( modules, {
 252                         browserstack,
 253                         esm,
 254                         jsdom: browser.browser === "jsdom",
 255                         port,
 256                         reportId
 257                 } );
 258
 259                 const options = {
 260                         browserstack,
 261                         concurrency,
 262                         debug,
 263                         headless,
 264                         modules,
 265                         reportId,
 266                         runId,
 267                         tunnelId,
 268                         verbose
 269                 };
 270
 271                 addRun( url, browser, options );
 272         }
 273
 274         for ( const browser of browsers ) {
 275                 if ( isolate ) {
 276                         for ( const module of modules ) {
 277                                 queueRun( [ module ], browser );
 278                         }
 279                 } else {
 280                         queueRun( modules, browser );
 281                 }
 282         }
 283
 284         try {
 285                 console.log( `Starting Run ${ runId }...` );
 286                 await runAll();
 287         } catch ( error ) {
 288                 console.error( error );
 289                 if ( !debug ) {
 290                         gracefulExit( 1 );
 291                 }
 292         } finally {
 293                 console.log();
 294                 if ( errorMessages.length === 0 ) {
 295                         let stop = false;
 296                         for ( const report of Object.values( reports ) ) {
 297                                 if ( !report.total ) {
 298                                         stop = true;
 299                                         console.error(
 300                                                 chalk.red(
 301                                                         `No tests were run for ${ report.modules.join(
 302                                                                 ", "
 303                                                         ) } in ${ getBrowserString( report.browser ) }`
 304                                                 )
 305                                         );
 306                                 }
 307                         }
 308                         if ( stop ) {
 309                                 return gracefulExit( 1 );
 310                         }
 311                         console.log( chalk.green( "All tests passed!" ) );
 312
 313                         if ( !debug || browserstack ) {
 314                                 gracefulExit( 0 );
 315                         }
 316                 } else {
 317                         console.error( chalk.red( `${ errorMessages.length } tests failed.` ) );
 318                         console.log(
 319                                 errorMessages.map( ( error, i ) => `\n${ i + 1 }. ${ error }` ).join( "\n" )
 320                         );
 321
 322                         if ( debug ) {
 323                                 console.log();
 324                                 if ( browserstack ) {
 325                                         console.log( "Leaving browsers with failures open for debugging." );
 326                                         console.log(
 327                                                 "View running sessions at https://automate.browserstack.com/dashboard/v2/"
 328                                         );
 329                                 } else {
 330                                         console.log( "Leaving browsers open for debugging." );
 331                                 }
 332                                 console.log( "Press Ctrl+C to exit." );
 333                         } else {
 334                                 gracefulExit( 1 );
 335                         }
 336                 }
 337         }
 338 }