buildSrc/src/main/java/cc/squirreljme/plugin/multivm/VMTestTaskAction.java

   1 // -*- Mode: Java; indent-tabs-mode: t; tab-width: 4 -*-
   2 // ---------------------------------------------------------------------------
   3 // SquirrelJME
   4 //     Copyright (C) Stephanie Gawroriski <xer@multiphasicapps.net>
   5 // ---------------------------------------------------------------------------
   6 // SquirrelJME is under the Mozilla Public License Version 2.0.
   7 // See license.mkd for licensing and copyright information.
   8 // ---------------------------------------------------------------------------
   9
  10 package cc.squirreljme.plugin.multivm;
  11
  12 import cc.squirreljme.plugin.SquirrelJMEPluginConfiguration;
  13 import cc.squirreljme.plugin.multivm.ident.SourceTargetClassifier;
  14 import cc.squirreljme.plugin.util.JavaExecSpecFiller;
  15 import cc.squirreljme.plugin.util.SerializedPath;
  16 import cc.squirreljme.plugin.util.SimpleJavaExecSpecFiller;
  17 import java.io.BufferedReader;
  18 import java.io.IOException;
  19 import java.io.InputStreamReader;
  20 import java.io.PrintStream;
  21 import java.nio.charset.StandardCharsets;
  22 import java.nio.file.Files;
  23 import java.nio.file.Path;
  24 import java.nio.file.StandardOpenOption;
  25 import java.util.ArrayList;
  26 import java.util.Arrays;
  27 import java.util.LinkedHashMap;
  28 import java.util.LinkedHashSet;
  29 import java.util.List;
  30 import java.util.Map;
  31 import java.util.Objects;
  32 import java.util.Set;
  33 import java.util.TreeMap;
  34 import java.util.UUID;
  35 import org.gradle.api.Action;
  36 import org.gradle.api.Task;
  37 import org.gradle.api.logging.Logger;
  38 import org.gradle.api.tasks.testing.Test;
  39 import org.gradle.workers.WorkQueue;
  40 import org.gradle.workers.WorkerExecutor;
  41
  42 /**
  43  * Performs the testing of the program.
  44  *
  45  * This is responsible for executing the test and outputting JUnit results as
  46  * well which record the test logs accordingly. This also will record snapshots
  47  * for profiling and otherwise, if possible.
  48  *
  49  * Entry is via {@code net.multiphasicapps.tac.MainSingleRunner} which is
  50  * passed the test to load and execute.
  51  *
  52  * @since 2020/08/07
  53  */
  54 public class VMTestTaskAction
  55         implements Action<Task>
  56 {
  57
  58         /** The special key for quick finding test results. */
  59         static final String _SPECIAL_KEY =
  60                 "XERSQUIRRELJMEXER";
  61
  62         /** Print the test manifest? */
  63         public static final String PRINT_TEST_MANIFEST =
  64                 "net.multiphasicapps.tac.resultManifest";
  65
  66         /** The maximum parallel tests that can run at once. */
  67         private static final int _MAX_PARALLEL_TESTS =
  68                 4;
  69
  70         /** Cached CPU count. */
  71         private static volatile int _CACHED_CPU_COUNT;
  72
  73         /** The worker executor. */
  74         protected final WorkerExecutor executor;
  75
  76         /** The classifier used. */
  77         protected final SourceTargetClassifier classifier;
  78
  79         /**
  80          * Initializes the virtual machine task action.
  81          *
  82          * @param __executor The executor for tasks.
  83          * @param __classifier The classifier used.
  84          * @throws NullPointerException On null arguments.
  85          * @since 2020/08/23
  86          */
  87         public VMTestTaskAction(WorkerExecutor __executor,
  88                 SourceTargetClassifier __classifier)
  89                 throws NullPointerException
  90         {
  91                 if (__executor == null || __classifier == null)
  92                         throw new NullPointerException("NARG");
  93
  94                 this.executor = __executor;
  95                 this.classifier = __classifier;
  96         }
  97
  98         /**
  99          * {@inheritDoc}
 100          * @since 2020/08/07
 101          */
 102         @Override
 103         public void execute(Task __task)
 104         {
 105                 // The task used for testing
 106                 Test testTask = (Test)__task;
 107
 108                 // Debug
 109                 Logger logger = __task.getLogger();
 110                 logger.debug("Tests: {}", VMHelpers.runningTests(
 111                         __task.getProject(), this.classifier.getSourceSet()));
 112
 113                 // We want our tasks to run from within Gradle
 114                 WorkQueue queue = this.executor.noIsolation();
 115
 116                 // All results will go here
 117                 String sourceSet = this.classifier.getSourceSet();
 118                 VMSpecifier vmType = this.classifier.getVmType();
 119                 Path resultDir = VMHelpers.testResultXmlDir(__task.getProject(),
 120                         this.classifier).get();
 121
 122                 // All the result files will be read afterwards to determine whether
 123                 // this task will pass or fail
 124                 Map<String, Path> xmlResults = new TreeMap<>();
 125
 126                 // How many tests should be run be at once?
 127                 int maxParallel = VMTestTaskAction.maxParallelTests();
 128
 129                 // Determine the number of tests
 130                 Map<String, CandidateTestFiles> tests = VMHelpers.runningTests(
 131                         __task.getProject(), sourceSet).tests;
 132                 Set<String> testNames = tests.keySet();
 133                 int numTests = testNames.size();
 134
 135                 // Calculate suite run parameters
 136                 SuiteRunParameters runSuite = VMTestTaskAction.runSuite(
 137                         (VMBaseTask)__task, this.classifier);
 138
 139                 // Force non-parallel?
 140                 if (runSuite.noParallelTests)
 141                         maxParallel = 1;
 142
 143                 // We only need to set the classpath once
 144                 Path[] classPath = SerializedPath.unboxPaths(runSuite.classPath);
 145
 146                 // Debug
 147                 logger.debug("Testing ClassPath: {}",
 148                         Arrays.asList(classPath));
 149
 150                 // Make unique ID for logger binding for this session
 151                 String uniqueID = runSuite.uniqueId;
 152
 153                 // Setup logger for this session
 154                 __LogHolder__ logHolder = new __LogHolder__(logger);
 155                 VMTestWorkAction._LOGGERS.put(uniqueID, logHolder);
 156
 157                 // Execute the tests concurrently but up to the limit, as testing is
 158                 // very intense on CPU
 159                 int runCount = 0;
 160                 int submitCount = 0;
 161                 for (String testName : testNames)
 162                 {
 163                         // Calculate test running parameters
 164                         CandidateTestFiles candidate = tests.get(testName);
 165                         TestRunParameters runTest = VMTestTaskAction.runTest(
 166                                 (VMBaseTask)__task, this.classifier, runSuite, testName,
 167                                 candidate);
 168
 169                         // Where will the results be read from?
 170                         Path xmlResult = resultDir.resolve(
 171                                 VMHelpers.testResultXmlName(testName));
 172                         xmlResults.put(testName, xmlResult);
 173
 174                         // Which test number is this?
 175                         int submitId = ++submitCount;
 176
 177                         // Submit our work task which should be a simple JVM execute due
 178                         // to the limitations of Gradle workers
 179                         queue.submit(VMTestWorkAction.class, __params ->
 180                                 {
 181                                         // The logger used
 182                                         __params.getUniqueId().set(uniqueID);
 183
 184                                         // The test and where the results will go
 185                                         __params.getTestName().set(testName);
 186                                         __params.getResultFile().set(xmlResult.toFile());
 187
 188                                         // The command line to execute
 189                                         __params.getCommandLine().set(runTest.getCommandLine());
 190
 191                                         // Name of the VM for hostname setting
 192                                         __params.getVmName()
 193                                                 .set(vmType.vmName(VMNameFormat.PROPER_NOUN));
 194
 195                                         // Used for progress tracking
 196                                         __params.getCount().set(submitId);
 197                                         __params.getTotal().set(numTests);
 198                                 });
 199
 200                         // Already requested the number of tests to run, so let them
 201                         // finish first
 202                         if (++runCount >= maxParallel)
 203                         {
 204                                 // Reset counter so that we can trigger this again
 205                                 runCount = 0;
 206
 207                                 // Wait for the current set to finish
 208                                 queue.await();
 209                         }
 210                 }
 211
 212                 // Wait for the queue to finish
 213                 queue.await();
 214
 215                 // Get the status of every test
 216                 Map<String, ResultantTestInfo> testResults =
 217                         this.__testResults(xmlResults);
 218
 219                 // Determine and print any test failures
 220                 Set<String> failedTests = new LinkedHashSet<>();
 221                 for (ResultantTestInfo test : testResults.values())
 222                         if (test.result == VMTestResult.FAIL)
 223                         {
 224                                 failedTests.add(test.name);
 225
 226                                 logger.error("Failed test: {}", test.name);
 227                         }
 228
 229                 // Determine and ensure the directory where CSVs go exist
 230                 Path csvDir = VMHelpers.testResultsCsvDir(__task.getProject(),
 231                         this.classifier).get();
 232                 try
 233                 {
 234                         Files.createDirectories(csvDir);
 235                 }
 236
 237                 // Ignore failures here
 238                 catch (IOException e)
 239                 {
 240                         e.printStackTrace();
 241                 }
 242
 243                 // Print a CSV of all the test results
 244                 try (PrintStream ps = new PrintStream(Files.newOutputStream(
 245                         csvDir.resolve(VMHelpers.testResultsCsvName(__task.getProject())),
 246                                 StandardOpenOption.WRITE, StandardOpenOption.CREATE,
 247                                 StandardOpenOption.TRUNCATE_EXISTING),
 248                                 true, "utf-8"))
 249                 {
 250                         // Print CSV Header
 251                         ps.println("name,status,duration,simpleDuration");
 252
 253                         // Print each test
 254                         long totalTime = 0;
 255                         for (ResultantTestInfo e : testResults.values())
 256                         {
 257                                 totalTime += e.nanoseconds;
 258
 259                                 ps.printf("%s,%s,%d,%s%n",
 260                                         e.name, e.result.name(), e.nanoseconds,
 261                                         VMTestTaskAction.__simpleDuration(e.nanoseconds));
 262                         }
 263
 264                         // Put out totals
 265                         ps.printf("TOTAL,,%d,%s%n",
 266                                 totalTime, VMTestTaskAction.__simpleDuration(totalTime));
 267
 268                         // Ensure everything is written
 269                         ps.flush();
 270                 }
 271
 272                 // Ignore these failures
 273                 catch (IOException e)
 274                 {
 275                         e.printStackTrace();
 276                 }
 277
 278                 // Wipe logger session
 279                 VMTestWorkAction._LOGGERS.remove(uniqueID);
 280
 281                 // If there were failures, then fail this task with an exception
 282                 if (!failedTests.isEmpty())
 283                         throw new RuntimeException(
 284                                 "There were failing tests: " + failedTests);
 285         }
 286
 287         /**
 288          * Goes through all the XML files obtains the test results.
 289          *
 290          * @param __xmlResults The result paths for tests.
 291          * @return The mapping of all tests.
 292          * @throws NullPointerException On null arguments.
 293          * @since 2020/09/08
 294          */
 295         Map<String, ResultantTestInfo> __testResults(
 296                 Map<String, Path> __xmlResults)
 297                 throws NullPointerException
 298         {
 299                 if (__xmlResults == null)
 300                         throw new NullPointerException("NARG");
 301
 302                 // Colon positions
 303                 List<Integer> colons = new ArrayList<>(4);
 304
 305                 // Failure sequence
 306                 Map<String, ResultantTestInfo> result = new TreeMap<>();
 307                 for (Map.Entry<String, Path> test : __xmlResults.entrySet())
 308                         try
 309                         {
 310                                 // Resulting status and key
 311                                 VMTestResult testResult = null;
 312                                 long nanoseconds = -1;
 313
 314                                 // Check all lines of the file and see if one is found
 315                                 for (String line : Files.readAllLines(test.getValue()))
 316                                 {
 317                                         // Locate the special key
 318                                         int keyDx = line.indexOf(VMTestTaskAction._SPECIAL_KEY);
 319                                         if (keyDx < 0)
 320                                                 continue;
 321
 322                                         // Find indexes of all the colons after this key
 323                                         colons.clear();
 324                                         for (int at = keyDx; at >= 0;)
 325                                         {
 326                                                 at = line.indexOf(':', at + 1);
 327
 328                                                 if (keyDx > 0)
 329                                                         colons.add(at);
 330                                         }
 331
 332                                         // Need three colons here
 333                                         if (colons.size() < 3)
 334                                                 continue;
 335
 336                                         // Extract the key and value
 337                                         String key = line.substring(
 338                                                 colons.get(0) + 1, colons.get(1));
 339                                         String val = line.substring(
 340                                                 colons.get(1) + 1, colons.get(2));
 341
 342                                         // Depends on the key/value
 343                                         try
 344                                         {
 345                                                 switch (key)
 346                                                 {
 347                                                         case "result":
 348                                                                 testResult = VMTestResult.valueOf(val);
 349                                                                 break;
 350
 351                                                         case "nanoseconds":
 352                                                                 nanoseconds = Long.parseLong(val);
 353                                                                 break;
 354                                                 }
 355                                         }
 356                                         catch (IllegalArgumentException e)
 357                                         {
 358                                                 e.printStackTrace();
 359                                         }
 360                                 }
 361
 362                                 // Store result here
 363                                 result.put(test.getKey(), new ResultantTestInfo(test.getKey(),
 364                                         testResult, nanoseconds));
 365                         }
 366                         catch (IOException e)
 367                         {
 368                                 throw new RuntimeException("Could not read test XML.", e);
 369                         }
 370
 371                 return result;
 372         }
 373
 374         /**
 375          * Is debugging being used?
 376          *
 377          * @return If debugging is being used.
 378          * @since 2022/09/15
 379          */
 380         public static boolean isDebugging()
 381         {
 382                 String jdwpProp = System.getProperty("squirreljme.xjdwp",
 383                         System.getProperty("squirreljme.jdwp"));
 384                 return (jdwpProp != null && !jdwpProp.isEmpty());
 385         }
 386
 387         /**
 388          * Sets the maximum number of parallel tests to run.
 389          *
 390          * @return The max parallel tests to run at once.
 391          * @since 2022/09/11
 392          */
 393         public static int maxParallelTests()
 394         {
 395                 // If debugging, do not run in parallel
 396                 if (null != System.getProperty("squirreljme.xjdwp",
 397                         System.getProperty("squirreljme.jdwp")))
 398                         return 1;
 399
 400                 int cpuCount = VMTestTaskAction.physicalProcessorCount();
 401                 return (cpuCount <= 1 ? 1 :
 402                         Math.min(Math.max(2, cpuCount),
 403                                 VMTestTaskAction._MAX_PARALLEL_TESTS));
 404         }
 405
 406         /**
 407          * Returns the physical processor count.
 408          *
 409          * @return The physical processor count.
 410          * @since 2020/11/25
 411          */
 412         @SuppressWarnings("CallToSystemGetenv")
 413         public static int physicalProcessorCount()
 414         {
 415                 // Use pre-cached value if it is already known
 416                 int rv = VMTestTaskAction._CACHED_CPU_COUNT;
 417                 if (rv > 0)
 418                         return rv;
 419
 420                 // If this variable is set, just force all CPUs to be used
 421                 if (Boolean.parseBoolean(System.getenv("USE_ALL_PROCESSORS")))
 422                         return Math.max(1, Runtime.getRuntime().availableProcessors());
 423
 424                 // We need this so we can make a better guess of our current system
 425                 String osName = System.getProperty("os.name").toLowerCase();
 426
 427                 // Running on Windows
 428                 if (rv <= 0 && osName.contains("windows"))
 429                         rv = VMTestTaskAction.__cpuCountOnWindows();
 430
 431                 // Fallback to a generic CPU count
 432                 if (rv <= 0)
 433                         rv = Math.max(1, Runtime.getRuntime().availableProcessors() / 2);
 434
 435                 // Cache it and use it
 436                 VMTestTaskAction._CACHED_CPU_COUNT = rv;
 437                 return rv;
 438         }
 439
 440         /**
 441          * Initializes the suite parameters.
 442          *
 443          * @param __task The task.
 444          * @param __classifier The classifier used.
 445          * @return The parameters to run all tests with.
 446          * @throws NullPointerException On null arguments.
 447          * @since 2022/09/11
 448          */
 449         public static SuiteRunParameters runSuite(VMBaseTask __task,
 450                 SourceTargetClassifier __classifier)
 451                 throws NullPointerException
 452         {
 453                 if (__task == null || __classifier == null)
 454                         throw new NullPointerException("NARG");
 455
 456                 // Setup builder
 457                 SuiteRunParameters.SuiteRunParametersBuilder builder =
 458                         SuiteRunParameters.builder();
 459
 460                 // Determine system properties to use for testing
 461                 Map<String, String> sysProps = new LinkedHashMap<>();
 462                 if (Boolean.getBoolean("java.awt.headless"))
 463                         sysProps.put("java.awt.headless", "true");
 464
 465                 // Any specific changes to how tests run
 466                 SquirrelJMEPluginConfiguration config =
 467                         SquirrelJMEPluginConfiguration.configurationOrNull(
 468                                 __task.getProject());
 469                 if (config != null)
 470                 {
 471                         // If we define any system properties specifically for tests then
 472                         // use them here. Could be used for debugging.
 473                         sysProps.putAll(config.testSystemProperties);
 474
 475                         // Disable parallelism for these tests?
 476                         builder.noParallelTests(config.noParallelTests);
 477                 }
 478
 479                 // Can we directly refer to the emulator library already?
 480                 Path emuLib = VMHelpers.findEmulatorLib(__task.getProject());
 481                 if (emuLib != null && Files.exists(emuLib))
 482                         sysProps.put("squirreljme.emulator.libpath", emuLib.toString());
 483
 484                 // Setup parameters and build
 485                 return builder
 486                         .sysProps(sysProps)
 487                         .emuLib(new SerializedPath(emuLib))
 488                         .uniqueId(UUID.randomUUID().toString())
 489                         .classPath(SerializedPath.boxPaths(VMHelpers.runClassPath(
 490                                 __task, __classifier, true)))
 491                         .build();
 492         }
 493
 494         /**
 495          * Determines the command line that is used to run tests.
 496          *
 497          * @param __task The task this is for.
 498          * @param __classifier The classifier used.
 499          * @param __runSuite The existing run suite.
 500          * @param __testName The name of the test.
 501          * @param __candidate The test candidate, for test information.
 502          * @return The parameters for running this test.
 503          * @throws NullPointerException On null arguments.
 504          * @since 2022/09/11
 505          */
 506         public static TestRunParameters runTest(VMBaseTask __task,
 507                 SourceTargetClassifier __classifier,
 508                 SuiteRunParameters __runSuite, String __testName,
 509                 CandidateTestFiles __candidate)
 510                 throws NullPointerException
 511         {
 512                 if (__task == null || __classifier == null ||
 513                         __runSuite == null || __testName == null || __candidate == null)
 514                         throw new NullPointerException("NARG");
 515
 516                 TestRunParameters.TestRunParametersBuilder builder =
 517                         TestRunParameters.builder();
 518
 519                 // Default arguments, could be replaced by a proxy main
 520                 String mainClass = VMHelpers.SINGLE_TEST_RUNNER;
 521                 String[] mainArgs = new String[]{__testName};
 522
 523                 // Are we going to use a different proxy main class for this?
 524                 String proxyMain = __candidate.expectedValues.get("proxy-main");
 525                 if (proxyMain != null && !proxyMain.trim().isEmpty())
 526                 {
 527                         mainArgs = new String[]{mainClass, mainArgs[0]};
 528                         mainClass = proxyMain.trim();
 529                 }
 530
 531                 // Deserialize classpath
 532                 Path[] classPath = SerializedPath.unboxPaths(__runSuite.classPath);
 533
 534                 Map<String, String> sysProps = new LinkedHashMap<>(
 535                         __runSuite.getSysProps());
 536                 if (Boolean.parseBoolean(
 537                         __candidate.expectedValues.get("test-vm-target")))
 538                         sysProps.put("cc.squirreljme.test.vm",
 539                                 __classifier.getBangletVariant().banglet);
 540
 541                 // Print test result manifest?
 542                 if (Boolean.getBoolean(VMTestTaskAction.PRINT_TEST_MANIFEST) ||
 543                         (__task.hasProperty(VMTestTaskAction.PRINT_TEST_MANIFEST) &&
 544                         Boolean.parseBoolean(Objects.toString(
 545                                 __task.property(VMTestTaskAction.PRINT_TEST_MANIFEST)))))
 546                         sysProps.put(VMTestTaskAction.PRINT_TEST_MANIFEST, "true");
 547
 548                 // Determine the arguments that are used to spawn the JVM
 549                 JavaExecSpecFiller execSpec = new SimpleJavaExecSpecFiller();
 550                 __classifier.getVmType().spawnJvmArguments(__task.getProject(),
 551                         __classifier, true,
 552                         execSpec, mainClass, __testName, sysProps,
 553                         classPath, classPath, mainArgs);
 554
 555                 // Get command line
 556                 List<String> commandLine = new ArrayList<>();
 557                 for (String arg : execSpec.getCommandLine())
 558                         commandLine.add(arg);
 559
 560                 // Build final result
 561                 return builder
 562                         .commandLine(commandLine)
 563                         .build();
 564         }
 565
 566         /**
 567          * Attempts to get the physical CPU count on Windows.
 568          *
 569          * @return The physical CPU count on Windows, {@code 0} is returned if it
 570          * could not be obtained.
 571          * @since 2020/11/25
 572          */
 573         @SuppressWarnings("UseOfProcessBuilder")
 574         private static int __cpuCountOnWindows()
 575         {
 576                 // This information could be obtained by running a specific program
 577                 try
 578                 {
 579                         // Spawn new process that will contain the CPU count
 580                         Process proc = new ProcessBuilder()
 581                                 .command("cmd", "/C", "WMIC", "CPU", "Get", "/Format:List")
 582                                 .redirectError(ProcessBuilder.Redirect.INHERIT)
 583                                 .start();
 584
 585                         // Let the process run accordingly
 586                         try
 587                         {
 588                                 // Command failed to start, so likely unreliable
 589                                 if (proc.waitFor() != 0)
 590                                         return 0;
 591                         }
 592
 593                         // We probably want the process to go away if this happens
 594                         catch (InterruptedException ignored)
 595                         {
 596                                 proc.destroyForcibly();
 597
 598                                 // The CPU count will be invalid here
 599                                 return 0;
 600                         }
 601
 602                         // Look for the CPU information lines
 603                         int numCores = 0;
 604                         try (BufferedReader br = new BufferedReader(new InputStreamReader(
 605                                 proc.getInputStream(), StandardCharsets.UTF_8)))
 606                         {
 607                                 for (;;)
 608                                 {
 609                                         String ln = br.readLine();
 610
 611                                         if (ln == null)
 612                                                 break;
 613
 614                                         // Number of cores in the system
 615                                         if (ln.startsWith("NumberOfCores="))
 616                                                 numCores = Integer.parseInt(
 617                                                         ln.substring(ln.indexOf('=') + 1));
 618                                 }
 619                         }
 620
 621                         // Were we able to glean the CPU count?
 622                         return Math.max(numCores, 0);
 623                 }
 624
 625                 // Ignore failures
 626                 catch (IOException|NumberFormatException e)
 627                 {
 628                         // Toss exception
 629                         new RuntimeException("Could not determine physical CPU count.", e)
 630                                 .printStackTrace();
 631
 632                         return 0;
 633                 }
 634         }
 635
 636         /**
 637          * Returns the simple duration of the test.
 638          *
 639          * @param __dur The nanoseconds to map.
 640          * @return The simple duration string.
 641          * @since 2020/11/26
 642          */
 643         @SuppressWarnings("MagicNumber")
 644         private static String __simpleDuration(long __dur)
 645         {
 646                 // Instantly finished?
 647                 if (__dur <= 0)
 648                         return "instant";
 649
 650                 StringBuilder sb = new StringBuilder();
 651
 652                 // Nanoseconds
 653                 long ns = __dur % 1_000_000L;
 654                 sb.insert(0, String.format("%dns", ns));
 655                 __dur /= 1_000_000L;
 656
 657                 // Milliseconds
 658                 long ms = __dur % 1_000L;
 659                 if (ms > 0)
 660                         sb.insert(0, String.format("%dms ", ms));
 661                 __dur /= 1_000L;
 662
 663                 // Seconds
 664                 long s = __dur % 60L;
 665                 if (s > 0)
 666                         sb.insert(0, String.format("%ds ", s));
 667                 __dur /= 60L;
 668
 669                 // Minutes
 670                 long m = __dur & 60L;
 671                 if (m > 0)
 672                         sb.insert(0, String.format("%dm ", m));
 673
 674                 return sb.toString();
 675         }
 676 }