buildSrc/src/main/java/cc/squirreljme/plugin/multivm/VMTestFrameworkTestClassProcessor.java

   1 // -*- Mode: Java; indent-tabs-mode: t; tab-width: 4 -*-
   2 // ---------------------------------------------------------------------------
   3 // Multi-Phasic Applications: SquirrelJME
   4 //     Copyright (C) Stephanie Gawroriski <xer@multiphasicapps.net>
   5 // ---------------------------------------------------------------------------
   6 // SquirrelJME is under the GNU General Public License v3+, or later.
   7 // See license.mkd for licensing and copyright information.
   8 // ---------------------------------------------------------------------------
   9
  10 package cc.squirreljme.plugin.multivm;
  11
  12 import cc.squirreljme.plugin.util.TestResultOutputStream;
  13 import java.io.IOException;
  14 import java.nio.charset.StandardCharsets;
  15 import java.util.ArrayList;
  16 import java.util.List;
  17 import java.util.Map;
  18 import java.util.Set;
  19 import java.util.TreeMap;
  20 import java.util.TreeSet;
  21 import java.util.concurrent.TimeUnit;
  22 import java.util.concurrent.atomic.AtomicReference;
  23 import java.util.function.Consumer;
  24 import org.gradle.api.internal.tasks.testing.DefaultTestClassDescriptor;
  25 import org.gradle.api.internal.tasks.testing.DefaultTestDescriptor;
  26 import org.gradle.api.internal.tasks.testing.DefaultTestMethodDescriptor;
  27 import org.gradle.api.internal.tasks.testing.DefaultTestSuiteDescriptor;
  28 import org.gradle.api.internal.tasks.testing.TestClassProcessor;
  29 import org.gradle.api.internal.tasks.testing.TestClassRunInfo;
  30 import org.gradle.api.internal.tasks.testing.TestCompleteEvent;
  31 import org.gradle.api.internal.tasks.testing.TestResultProcessor;
  32 import org.gradle.api.internal.tasks.testing.TestStartEvent;
  33 import org.gradle.api.tasks.testing.TestOutputEvent;
  34 import org.gradle.api.tasks.testing.TestResult;
  35 import org.gradle.internal.id.IdGenerator;
  36
  37 /**
  38  * Processor for test classes.
  39  *
  40  * @since 2022/09/11
  41  */
  42 public class VMTestFrameworkTestClassProcessor
  43         implements TestClassProcessor
  44 {
  45         /** Tests to run. */
  46         protected final Map<String, Set<VMTestFrameworkTestClass>> runTests =
  47                 new TreeMap<>();
  48
  49         /** Test result output. */
  50         protected final AtomicReference<TestResultProcessor> resultProcessor =
  51                 new AtomicReference<>();
  52
  53         /** The tests that are available. */
  54         protected final Map<String, CandidateTestFiles> availableTests;
  55
  56         /** The ID generator to use. */
  57         protected final IdGenerator<?> idGenerator;
  58
  59         /** The name of this project. */
  60         protected final String projectName;
  61
  62         /** The run parameters. */
  63         protected final Map<String, TestRunParameters> runParameters;
  64
  65         /** The thread that is running tests. */
  66         private volatile Thread _runningThread;
  67
  68         /** Stop running tests? */
  69         private volatile boolean _stopNow;
  70
  71         /** The final test result. */
  72         private volatile TestResult.ResultType _finalResult =
  73                 TestResult.ResultType.SKIPPED;
  74
  75         /**
  76          * Initializes the processor.
  77          *
  78          * @param __availableTests The tests that are available.
  79          * @param __projectName The name of the project.
  80          * @param __runParameters The run parameters.
  81          * @throws NullPointerException On null arguments.
  82          * @since 2022/09/11
  83          */
  84         public VMTestFrameworkTestClassProcessor(
  85                 Map<String, CandidateTestFiles> __availableTests,
  86                 IdGenerator<?> __idGenerator, String __projectName,
  87                 Map<String, TestRunParameters> __runParameters)
  88                 throws NullPointerException
  89         {
  90                 if (__availableTests == null || __idGenerator == null ||
  91                         __projectName == null || __runParameters == null)
  92                         throw new NullPointerException("NARG");
  93
  94                 this.availableTests = __availableTests;
  95                 this.idGenerator = __idGenerator;
  96                 this.projectName = __projectName;
  97                 this.runParameters = __runParameters;
  98         }
  99
 100         /**
 101          * {@inheritDoc}
 102          * @since 2022/09/11
 103          */
 104         @Override
 105         public void startProcessing(TestResultProcessor __resultProcessor)
 106         {
 107                 // Store this for late
 108                 synchronized (this.resultProcessor)
 109                 {
 110                         this.resultProcessor.set(__resultProcessor);
 111                 }
 112         }
 113
 114         /**
 115          * {@inheritDoc}
 116          * @since 2022/09/11
 117          */
 118         @Override
 119         public void processTestClass(TestClassRunInfo __testClass)
 120         {
 121                 // Build test name for later usage
 122                 String test = __testClass.getTestClassName();
 123                 VMTestFrameworkTestClass testClass =
 124                         new VMTestFrameworkTestClass(test);
 125
 126                 // Was there a match at all?
 127                 boolean match = false;
 128
 129                 Map<String, Set<VMTestFrameworkTestClass>> runTests = this.runTests;
 130                 for (String availableTest : this.availableTests.keySet())
 131                 {
 132                         VMTestFrameworkTestClass available =
 133                                 new VMTestFrameworkTestClass(availableTest);
 134
 135                         // Belong to a different class, ignore completely
 136                         if (!testClass.className.equals(available.className))
 137                                 continue;
 138
 139                         // Is this pure match?
 140                         boolean currentMatch = false;
 141                         currentMatch = testClass.normal.equals(available.normal);
 142
 143                         // Is there a variant match?
 144                         if (!currentMatch && testClass.variant != null)
 145                                 currentMatch = testClass.variant.equals(available.variant);
 146
 147                         // We have a primary sub-variant which matches, but we asked for no
 148                         // secondary sub-variant and there is one... we want to grab it
 149                         if (!currentMatch && testClass.primarySubVariant != null)
 150                                 currentMatch = testClass.primarySubVariant
 151                                         .equals(available.primarySubVariant) &&
 152                                         testClass.secondarySubVariant == null &&
 153                                         available.secondarySubVariant != null;
 154
 155                         // We did not ask for a variant but there is one, so include the
 156                         // test
 157                         if (!currentMatch && testClass.variant == null &&
 158                                 available.variant != null)
 159                                 currentMatch = true;
 160
 161                         // Matched, so add the test
 162                         if (currentMatch)
 163                                 runTests.computeIfAbsent(testClass.className,
 164                                                 (__k) -> new TreeSet<>())
 165                                         .add(new VMTestFrameworkTestClass(availableTest));
 166
 167                         // Matching emits a match
 168                         match |= currentMatch;
 169                 }
 170
 171                 // If there is no match and a variant was specified, we likely want
 172                 // a slightly different parameter
 173                 if (!match && testClass.variant != null)
 174                 {
 175                         // Remember class for later, sort by classes all together
 176                         runTests.computeIfAbsent(testClass.className,
 177                                 (__k) -> new TreeSet<>()).add(testClass);
 178                 }
 179         }
 180
 181         /**
 182          * {@inheritDoc}
 183          * @since 2022/09/11
 184          */
 185         @Override
 186         public void stop()
 187         {
 188                 // Stop is a bit of a misnomer, it means stop processing and then run
 189                 // all the tests...
 190
 191                 // Remember thread for later stop
 192                 synchronized (this)
 193                 {
 194                         this._runningThread = Thread.currentThread();
 195                 }
 196
 197                 AtomicReference<TestResultProcessor> resultProcessor =
 198                         this.resultProcessor;
 199
 200                 // Suite for the entire project group
 201                 IdGenerator<?> idGenerator = this.idGenerator;
 202                 DefaultTestSuiteDescriptor suiteDesc =
 203                         new DefaultTestSuiteDescriptor(
 204                                 idGenerator.generateId(),
 205                                 this.projectName);
 206
 207                 // Suite has started
 208                 VMTestFrameworkTestClassProcessor.resultAction(resultProcessor,
 209                         (__rp) -> __rp.started(suiteDesc,
 210                         new TestStartEvent(System.currentTimeMillis())));
 211
 212                 // Go through and actually run all the tests
 213                 for (Map.Entry<String, Set<VMTestFrameworkTestClass>> byClass :
 214                         this.runTests.entrySet())
 215                 {
 216                         // Do not run any more classes?
 217                         synchronized (this)
 218                         {
 219                                 if (this._stopNow)
 220                                         break;
 221                         }
 222
 223                         // Start class test
 224                         DefaultTestClassDescriptor classDesc =
 225                                 new DefaultTestClassDescriptor(idGenerator.generateId(),
 226                                         byClass.getKey(), byClass.getKey());
 227                         VMTestFrameworkTestClassProcessor.resultAction(resultProcessor,
 228                                 (__rp) -> __rp.started(classDesc,
 229                                         new TestStartEvent(System.currentTimeMillis(),
 230                                                 suiteDesc.getId())));
 231
 232                         // Go through all tests in this class
 233                         TestResult.ResultType classResult = TestResult.ResultType.SKIPPED;
 234                         for (VMTestFrameworkTestClass testName : byClass.getValue())
 235                         {
 236                                 // Do not run any more tests?
 237                                 synchronized (this)
 238                                 {
 239                                         if (this._stopNow)
 240                                                 break;
 241                                 }
 242
 243                                 // We do not know how to run the test, so do nothing
 244                                 if (this.runParameters.get(testName.normal) == null)
 245                                         continue;
 246
 247                                 // Run test and get its result
 248                                 TestResult.ResultType result = this.__runSingleTest(suiteDesc,
 249                                         testName, classDesc);
 250
 251                                 // Update class based result
 252                                 classResult = VMTestFrameworkTestClassProcessor
 253                                         .calculateResult(classResult, result);
 254
 255                                 // Change the global suite test result here
 256                                 this._finalResult = VMTestFrameworkTestClassProcessor
 257                                         .calculateResult(this._finalResult, result);
 258                         }
 259
 260                         // Mark class as completed
 261                         TestResult.ResultType finalClassResult = classResult;
 262                         VMTestFrameworkTestClassProcessor.resultAction(resultProcessor,
 263                                 (__rp) -> __rp.completed(classDesc.getId(),
 264                                         new TestCompleteEvent(System.currentTimeMillis(),
 265                                                 finalClassResult)));
 266                 }
 267
 268                 // If failed, emit a throwable... if we do not do this then Gradle does
 269                 // not care if a test failed and will just continue on happily like
 270                 // nothing ever happened
 271                 TestResult.ResultType finalResult = this._finalResult;
 272                 if (finalResult == TestResult.ResultType.FAILURE)
 273                         VMTestFrameworkTestClassProcessor.resultAction(resultProcessor,
 274                                 (__rp) -> __rp.failure(suiteDesc.getId(),
 275                                         VMTestFrameworkTestClassProcessor
 276                                                 .messageThrow("Tests have failed.")));
 277
 278                 // Use the final result from all the test runs
 279                 VMTestFrameworkTestClassProcessor.resultAction(resultProcessor,
 280                         (__rp) -> __rp.completed(suiteDesc.getId(),
 281                                 new TestCompleteEvent(System.currentTimeMillis(),
 282                                         finalResult)));
 283         }
 284
 285         /**
 286          * {@inheritDoc}
 287          * @since 2022/09/11
 288          */
 289         @Override
 290         public void stopNow()
 291         {
 292                 // Signal that tests should stop
 293                 synchronized (this)
 294                 {
 295                         // Do stop now
 296                         this._stopNow = true;
 297
 298                         // Make this invalid
 299                         synchronized (this.resultProcessor)
 300                         {
 301                                 this.resultProcessor.set(null);
 302                         }
 303
 304                         // Interrupt thread quickly
 305                         Thread runningThread = this._runningThread;
 306                         if (runningThread != null)
 307                                 runningThread.interrupt();
 308
 309                         // Make sure to notify on all monitors
 310                         this.notifyAll();
 311                 }
 312         }
 313
 314         /**
 315          * Runs the test and gives the result of it.
 316          *
 317          * @param __suiteDesc The suite descriptor.
 318          * @param __testName The name of the test.
 319          * @param __classDesc The owning class descriptor, since this is a group.
 320          * @return The result of the test.
 321          * @since 2022/09/11
 322          */
 323         @SuppressWarnings("UseOfProcessBuilder")
 324         private TestResult.ResultType __runSingleTest(
 325                 DefaultTestSuiteDescriptor __suiteDesc,
 326                 VMTestFrameworkTestClass __testName,
 327                 DefaultTestClassDescriptor __classDesc)
 328         {
 329                 Map<String, TestRunParameters> runParameters = this.runParameters;
 330                 AtomicReference<TestResultProcessor> resultProcessor =
 331                         this.resultProcessor;
 332                 IdGenerator<?> idGenerator = this.idGenerator;
 333
 334                 // Default to failure
 335                 TestResult.ResultType testResult = TestResult.ResultType.FAILURE;
 336
 337                 // Threads for processing stream data
 338                 Process process = null;
 339                 Thread stdOutThread = null;
 340                 Thread stdErrThread = null;
 341
 342                 // Make sure process and thread are killed at the end
 343                 try
 344                 {
 345                         // Check to see if we are stopping testing
 346                         synchronized (this)
 347                         {
 348                                 // If we are forcing a stop, mark as failure
 349                                 if (this._stopNow)
 350                                         return TestResult.ResultType.FAILURE;
 351                         }
 352
 353                         // Get parameters for this test run
 354                         TestRunParameters runTest = runParameters.get(__testName.normal);
 355
 356                         // Setup process to run
 357                         ProcessBuilder builder = new ProcessBuilder();
 358
 359                         // The command we are executing
 360                         builder.command(runTest.getCommandLine());
 361
 362                         // Redirect all the outputs we have
 363                         builder.redirectOutput(ProcessBuilder.Redirect.PIPE);
 364                         builder.redirectError(ProcessBuilder.Redirect.PIPE);
 365
 366                         // Start method test
 367                         DefaultTestDescriptor methodDesc = new DefaultTestMethodDescriptor(
 368                                 idGenerator.generateId(), __testName.className,
 369                                 (__testName.variant == null ? "test" :
 370                                         String.format("test[%s]", __testName.variant)));
 371                         VMTestFrameworkTestClassProcessor.resultAction(resultProcessor,
 372                                 (__rp) -> __rp.started(methodDesc,
 373                                         new TestStartEvent(System.currentTimeMillis(),
 374                                                 __classDesc.getId())));
 375
 376                         // Start the process
 377                         try
 378                         {
 379                                 process = builder.start();
 380                         }
 381
 382                         // Failed to start?
 383                         catch (IOException __e)
 384                         {
 385                                 // Failed to start test
 386                                 VMTestFrameworkTestClassProcessor.resultAction(resultProcessor,
 387                                         (__rp) -> __rp.failure(__suiteDesc.getId(),
 388                                                 new Throwable("Failed to start test.")));
 389
 390                                 throw new RuntimeException(__e);
 391                         }
 392
 393                         // Setup listening buffer threads
 394                         VMTestOutputBuffer stdOut = new VMTestOutputBuffer(
 395                                 process.getInputStream(),
 396                                 new TestResultOutputStream(resultProcessor, methodDesc.getId(),
 397                                         TestOutputEvent.Destination.StdOut),
 398                                 false);
 399                         VMTestOutputBuffer stdErr = new VMTestOutputBuffer(
 400                                 process.getErrorStream(),
 401                                 new TestResultOutputStream(resultProcessor, methodDesc.getId(),
 402                                         TestOutputEvent.Destination.StdErr),
 403                                 true);
 404
 405                         // Setup threads for reading standard output and error
 406                         stdOutThread = new Thread(stdOut, "stdOutReader");
 407                         stdErrThread = new Thread(stdErr, "stdErrReader");
 408
 409                         // Start threads so console lines can be read as they appear
 410                         stdOutThread.start();
 411                         stdErrThread.start();
 412
 413                         // Run the test, default to failed exit code
 414                         int exitCode = -1;
 415                         boolean isDebugging = VMTestTaskAction.isDebugging();
 416                         for (long startTime = System.nanoTime();;)
 417                                 try
 418                                 {
 419                                         // How much time is left? If debugging time never runs
 420                                         // out
 421                                         long timeLeft = Math.max(VMTestWorkAction.TEST_TIMEOUT -
 422                                                 (System.nanoTime() - startTime),
 423                                                 (isDebugging ? 1000L : 0L));
 424
 425                                         // Did we run out of time?
 426                                         // Wait for however long this takes to complete
 427                                         if (timeLeft <= 0 || this._stopNow ||
 428                                                 process.waitFor(timeLeft, TimeUnit.NANOSECONDS))
 429                                         {
 430                                                 exitCode = process.exitValue();
 431                                                 break;
 432                                         }
 433                                 }
 434
 435                                 // We got interrupted or ran out of time, make sure the process
 436                                 // stops, and we continue with the result
 437                                 catch (IllegalThreadStateException|InterruptedException
 438                                         ignored)
 439                                 {
 440                                         // Forcibly destroy the process if it is alive
 441                                         if (process.isAlive())
 442                                                 process.destroyForcibly();
 443
 444                                         // Make sure the threads process their output
 445                                         stdOutThread.interrupt();
 446                                         stdErrThread.interrupt();
 447                                 }
 448
 449                         // Force completion of the read thread, we cannot continue if
 450                         // the other thread is currently working...
 451                         stdOut.getBytes(stdOutThread);
 452                         byte[] stdErrBytes = stdErr.getBytes(stdErrThread);
 453
 454                         // The result determines whether this succeeded, skipped, or
 455                         // failed
 456                         switch (exitCode)
 457                         {
 458                                 // Success for zero
 459                                 case 0:
 460                                         testResult = TestResult.ResultType.SUCCESS;
 461                                         break;
 462
 463                                 // Skipped is just two
 464                                 case 2:
 465                                         testResult = TestResult.ResultType.SKIPPED;
 466                                         break;
 467
 468                                 // Treat anything else as failure
 469                                 default:
 470                                         testResult = TestResult.ResultType.FAILURE;
 471                                         break;
 472                         }
 473
 474                         // If failed, emit a throwable...
 475                         TestResult.ResultType finalResult = testResult;
 476                         if (finalResult == TestResult.ResultType.FAILURE)
 477                                 VMTestFrameworkTestClassProcessor.resultAction(resultProcessor,
 478                                         (__rp) -> __rp.failure(methodDesc.getId(),
 479                                                 VMTestFrameworkTestClassProcessor
 480                                                         .messageThrow("Test failed: " +
 481                                                                 __testName.normal)));
 482
 483                         // Mark method as completed
 484                         VMTestFrameworkTestClassProcessor.resultAction(resultProcessor,
 485                                 (__rp) -> __rp.completed(methodDesc.getId(),
 486                                         new TestCompleteEvent(System.currentTimeMillis(),
 487                                                 finalResult)));
 488                 }
 489
 490                 // Interrupt read/write threads and kill the process if it is alive
 491                 finally
 492                 {
 493                         // If our test process is still alive, stop it
 494                         if (process != null)
 495                                 if (process.isAlive())
 496                                         process.destroyForcibly();
 497
 498                         // Stop the standard output thread from running
 499                         if (stdOutThread != null)
 500                                 stdOutThread.interrupt();
 501
 502                         // Stop the standard error thread from running
 503                         if (stdErrThread != null)
 504                                 stdErrThread.interrupt();
 505                 }
 506
 507                 return testResult;
 508         }
 509
 510         /**
 511          * Calculates the test result.
 512          *
 513          * @param __input The input result.
 514          * @param __modifier The modifier to the result.
 515          * @return The new result that should be used.
 516          * @since 2022/09/14
 517          */
 518         public static TestResult.ResultType calculateResult(
 519                 TestResult.ResultType __input, TestResult.ResultType __modifier)
 520         {
 521                 if (__input == null || __modifier == null)
 522                         throw new NullPointerException("NARG");
 523
 524                 // Depends on our target result
 525                 switch (__modifier)
 526                 {
 527                                 // Keep the old state, this does not cause a change
 528                         case SKIPPED:
 529                                 return __input;
 530
 531                                 // Change skipped to success, but nothing else
 532                         case SUCCESS:
 533                                 if (__input == TestResult.ResultType.SKIPPED)
 534                                         return __modifier;
 535                                 return __input;
 536
 537                                 // Otherwise, always mark failure
 538                         case FAILURE:
 539                                 return __modifier;
 540                 }
 541
 542                 throw new Error("OOPS");
 543         }
 544
 545         /**
 546          * Generates a throwable useful for printing the output.
 547          *
 548          * @param __output The output.
 549          * @return The throwable to use for the message.
 550          * @since 2022/09/14
 551          */
 552         public static Throwable messageThrow(byte[] __output)
 553         {
 554                 return VMTestFrameworkTestClassProcessor.messageThrow(
 555                         (__output == null || __output.length <= 0 ? "" :
 556                                 new String(__output, StandardCharsets.UTF_8)));
 557         }
 558
 559         /**
 560          * Generates a throwable useful for printing the output.
 561          *
 562          * @param __output The output.
 563          * @return The throwable to use for the message.
 564          * @since 2022/09/14
 565          */
 566         private static Throwable messageThrow(String __output)
 567         {
 568                 return new VMTestFrameworkThrowableOutput(__output);
 569         }
 570
 571         /**
 572          * Runs the given test result processor action, assuming that it is still
 573          * a valid one. When {@link #stopNow()} then any processor that did exist
 574          * must not be used ever again.
 575          *
 576          * @param __atom The atomic to check on.
 577          * @param __doThis Do this action.
 578          * @throws NullPointerException On null arguments.
 579          * @since 2022/09/11
 580          */
 581         @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
 582         public static void resultAction(
 583                 AtomicReference<TestResultProcessor> __atom,
 584                 Consumer<TestResultProcessor> __doThis)
 585                 throws NullPointerException
 586         {
 587                 if (__atom == null || __doThis == null)
 588                         throw new NullPointerException("NARG");
 589
 590                 synchronized (__atom)
 591                 {
 592                         TestResultProcessor processor = __atom.get();
 593                         if (processor != null)
 594                                 __doThis.accept(processor);
 595                 }
 596         }
 597 }