test/runner/run.js

   1 import chalk from "chalk";
   2 import { asyncExitHook, gracefulExit } from "exit-hook";
   3 import { getLatestBrowser } from "./browserstack/api.js";
   4 import { buildBrowserFromString } from "./browserstack/buildBrowserFromString.js";
   5 import { localTunnel } from "./browserstack/local.js";
   6 import { reportEnd, reportTest } from "./reporter.js";
   7 import { createTestServer } from "./createTestServer.js";
   8 import { buildTestUrl } from "./lib/buildTestUrl.js";
   9 import { generateHash, printModuleHashes } from "./lib/generateHash.js";
  10 import { getBrowserString } from "./lib/getBrowserString.js";
  11 import { modules as allModules } from "./flags/modules.js";
  12 import { cleanupAllBrowsers, touchBrowser } from "./browsers.js";
  13 import {
  14         addRun,
  15         getNextBrowserTest,
  16         hardRetryTest,
  17         retryTest,
  18         runAll
  19 } from "./queue.js";
  20
  21 const EXIT_HOOK_WAIT_TIMEOUT = 60 * 1000;
  22
  23 /**
  24  * Run modules in parallel in different browser instances.
  25  */
  26 export async function run( {
  27         browser: browserNames = [],
  28         browserstack,
  29         concurrency,
  30         debug,
  31         esm,
  32         hardRetries,
  33         headless,
  34         isolate,
  35         module: modules = [],
  36         retries = 0,
  37         runId,
  38         verbose
  39 } ) {
  40         if ( !browserNames.length ) {
  41                 browserNames = [ "chrome" ];
  42         }
  43         if ( !modules.length ) {
  44                 modules = allModules;
  45         }
  46         if ( headless && debug ) {
  47                 throw new Error(
  48                         "Cannot run in headless mode and debug mode at the same time."
  49                 );
  50         }
  51
  52         if ( verbose ) {
  53                 console.log( browserstack ? "Running in BrowserStack." : "Running locally." );
  54         }
  55
  56         const errorMessages = [];
  57         const pendingErrors = {};
  58
  59         // Convert browser names to browser objects
  60         let browsers = browserNames.map( ( b ) => ( { browser: b } ) );
  61         const tunnelId = generateHash(
  62                 `${ Date.now() }-${ modules.join( ":" ) }-${ ( browserstack || [] )
  63                         .concat( browserNames )
  64                         .join( ":" ) }`
  65         );
  66
  67         // A unique identifier for this run
  68         if ( !runId ) {
  69                 runId = tunnelId;
  70         }
  71
  72         // Create the test app and
  73         // hook it up to the reporter
  74         const reports = Object.create( null );
  75         const app = await createTestServer( async( message ) => {
  76                 switch ( message.type ) {
  77                         case "testEnd": {
  78                                 const reportId = message.id;
  79                                 const report = reports[ reportId ];
  80                                 touchBrowser( report.browser );
  81                                 const errors = reportTest( message.data, reportId, report );
  82                                 pendingErrors[ reportId ] ??= Object.create( null );
  83                                 if ( errors ) {
  84                                         pendingErrors[ reportId ][ message.data.name ] = errors;
  85                                 } else {
  86                                         const existing = pendingErrors[ reportId ][ message.data.name ];
  87
  88                                         // Show a message for flakey tests
  89                                         if ( existing ) {
  90                                                 console.log();
  91                                                 console.warn(
  92                                                         chalk.italic(
  93                                                                 chalk.gray( existing.replace( "Test failed", "Test flakey" ) )
  94                                                         )
  95                                                 );
  96                                                 console.log();
  97                                                 delete pendingErrors[ reportId ][ message.data.name ];
  98                                         }
  99                                 }
 100                                 break;
 101                         }
 102                         case "runEnd": {
 103                                 const reportId = message.id;
 104                                 const report = reports[ reportId ];
 105                                 touchBrowser( report.browser );
 106                                 const { failed, total } = reportEnd(
 107                                         message.data,
 108                                         message.id,
 109                                         reports[ reportId ]
 110                                 );
 111                                 report.total = total;
 112
 113                                 // Handle failure
 114                                 if ( failed ) {
 115                                         const retry = retryTest( reportId, retries );
 116
 117                                         // Retry if retryTest returns a test
 118                                         if ( retry ) {
 119                                                 return retry;
 120                                         }
 121
 122                                         // Return early if hardRetryTest returns true
 123                                         if ( await hardRetryTest( reportId, hardRetries ) ) {
 124                                                 return;
 125                                         }
 126                                         errorMessages.push( ...Object.values( pendingErrors[ reportId ] ) );
 127                                 }
 128
 129                                 // Run the next test
 130                                 return getNextBrowserTest( reportId );
 131                         }
 132                         case "ack": {
 133                                 const report = reports[ message.id ];
 134                                 touchBrowser( report.browser );
 135                                 break;
 136                         }
 137                         default:
 138                                 console.warn( "Received unknown message type:", message.type );
 139                 }
 140         } );
 141
 142         // Start up local test server
 143         let server;
 144         let port;
 145         await new Promise( ( resolve ) => {
 146
 147                 // Pass 0 to choose a random, unused port
 148                 server = app.listen( 0, () => {
 149                         port = server.address().port;
 150                         resolve();
 151                 } );
 152         } );
 153
 154         if ( !server || !port ) {
 155                 throw new Error( "Server not started." );
 156         }
 157
 158         if ( verbose ) {
 159                 console.log( `Server started on port ${ port }.` );
 160         }
 161
 162         function stopServer() {
 163                 return new Promise( ( resolve ) => {
 164                         server.close( () => {
 165                                 if ( verbose ) {
 166                                         console.log( "Server stopped." );
 167                                 }
 168                                 resolve();
 169                         } );
 170                 } );
 171         }
 172
 173         async function cleanup() {
 174                 console.log( "Cleaning up..." );
 175
 176                 await cleanupAllBrowsers( { verbose } );
 177
 178                 if ( tunnel ) {
 179                         await tunnel.stop();
 180                         if ( verbose ) {
 181                                 console.log( "Stopped BrowserStackLocal." );
 182                         }
 183                 }
 184         }
 185
 186         asyncExitHook(
 187                 async() => {
 188                         await cleanup();
 189                         await stopServer();
 190                 },
 191                 { wait: EXIT_HOOK_WAIT_TIMEOUT }
 192         );
 193
 194         // Start up BrowserStackLocal
 195         let tunnel;
 196         if ( browserstack ) {
 197                 if ( headless ) {
 198                         console.warn(
 199                                 chalk.italic(
 200                                         "BrowserStack does not support headless mode. Running in normal mode."
 201                                 )
 202                         );
 203                         headless = false;
 204                 }
 205
 206                 // Convert browserstack to browser objects.
 207                 // If browserstack is an empty array, fall back
 208                 // to the browsers array.
 209                 if ( browserstack.length ) {
 210                         browsers = browserstack.map( ( b ) => {
 211                                 if ( !b ) {
 212                                         return browsers[ 0 ];
 213                                 }
 214                                 return buildBrowserFromString( b );
 215                         } );
 216                 }
 217
 218                 // Fill out browser defaults
 219                 browsers = await Promise.all(
 220                         browsers.map( async( browser ) => {
 221
 222                                 // Avoid undici connect timeout errors
 223                                 await new Promise( ( resolve ) => setTimeout( resolve, 100 ) );
 224
 225                                 const latestMatch = await getLatestBrowser( browser );
 226                                 if ( !latestMatch ) {
 227                                         console.error(
 228                                                 chalk.red( `Browser not found: ${ getBrowserString( browser ) }.` )
 229                                         );
 230                                         gracefulExit( 1 );
 231                                 }
 232                                 return latestMatch;
 233                         } )
 234                 );
 235
 236                 tunnel = await localTunnel( tunnelId );
 237                 if ( verbose ) {
 238                         console.log( "Started BrowserStackLocal." );
 239
 240                         printModuleHashes( modules );
 241                 }
 242         }
 243
 244         function queueRun( modules, browser ) {
 245                 const fullBrowser = getBrowserString( browser, headless );
 246                 const reportId = generateHash( `${ modules.join( ":" ) } ${ fullBrowser }` );
 247                 reports[ reportId ] = { browser, headless, modules };
 248
 249                 const url = buildTestUrl( modules, {
 250                         browserstack,
 251                         esm,
 252                         jsdom: browser.browser === "jsdom",
 253                         port,
 254                         reportId
 255                 } );
 256
 257                 const options = {
 258                         browserstack,
 259                         concurrency,
 260                         debug,
 261                         headless,
 262                         modules,
 263                         reportId,
 264                         runId,
 265                         tunnelId,
 266                         verbose
 267                 };
 268
 269                 addRun( url, browser, options );
 270         }
 271
 272         for ( const browser of browsers ) {
 273                 if ( isolate ) {
 274                         for ( const module of modules ) {
 275                                 queueRun( [ module ], browser );
 276                         }
 277                 } else {
 278                         queueRun( modules, browser );
 279                 }
 280         }
 281
 282         try {
 283                 console.log( `Starting Run ${ runId }...` );
 284                 await runAll();
 285         } catch ( error ) {
 286                 console.error( error );
 287                 if ( !debug ) {
 288                         gracefulExit( 1 );
 289                 }
 290         } finally {
 291                 console.log();
 292                 if ( errorMessages.length === 0 ) {
 293                         let stop = false;
 294                         for ( const report of Object.values( reports ) ) {
 295                                 if ( !report.total ) {
 296                                         stop = true;
 297                                         console.error(
 298                                                 chalk.red(
 299                                                         `No tests were run for ${ report.modules.join(
 300                                                                 ", "
 301                                                         ) } in ${ getBrowserString( report.browser ) }`
 302                                                 )
 303                                         );
 304                                 }
 305                         }
 306                         if ( stop ) {
 307                                 return gracefulExit( 1 );
 308                         }
 309                         console.log( chalk.green( "All tests passed!" ) );
 310
 311                         if ( !debug || browserstack ) {
 312                                 gracefulExit( 0 );
 313                         }
 314                 } else {
 315                         console.error( chalk.red( `${ errorMessages.length } tests failed.` ) );
 316                         console.log(
 317                                 errorMessages.map( ( error, i ) => `\n${ i + 1 }. ${ error }` ).join( "\n" )
 318                         );
 319
 320                         if ( debug ) {
 321                                 console.log();
 322                                 if ( browserstack ) {
 323                                         console.log( "Leaving browsers with failures open for debugging." );
 324                                         console.log(
 325                                                 "View running sessions at https://automate.browserstack.com/dashboard/v2/"
 326                                         );
 327                                 } else {
 328                                         console.log( "Leaving browsers open for debugging." );
 329                                 }
 330                                 console.log( "Press Ctrl+C to exit." );
 331                         } else {
 332                                 gracefulExit( 1 );
 333                         }
 334                 }
 335         }
 336 }