Issue #1825592 by alexpott: Add --repeat and --die-on-fail to run-tests.sh to...

Issue #1825592 by alexpott: Add --repeat and --die-on-fail to run-tests.sh to make random test failure diagnosis easier.

core/modules/simpletest/config/simpletest.settings.yml

 clear_results: '1'
+die_on_fail: '0'
 httpauth:
   method: '1'
   password: ''

core/modules/simpletest/lib/Drupal/simpletest/TestBase.php

@@ -143,6 +143,13 @@ abstract class TestBase {
    */
   protected $originalSettings;
 
+  /**
+   * TRUE if die on fail enabled.
+   *
+   * @var boolean
+   */
+  protected $dieOnFail = FALSE;
+
   /**
    * Constructor for Test.
    *
@@ -223,6 +230,9 @@ protected function assert($status, $message = '', $group = 'Other', array $calle
       return TRUE;
     }
     else {
+      if ($this->dieOnFail) {
+        exit(1);
+      }
       return FALSE;
     }
   }
@@ -660,6 +670,7 @@ public function run(array $methods = array()) {
       }
       $this->verboseClassName = str_replace("\\", "_", $class);
     }
+    $this->dieOnFail = $simpletest_config->get('die_on_fail');
     // HTTP auth settings (<username>:<password>) for the simpletest browser
     // when sending requests to the test site.
     $this->httpauth_method = (int) $simpletest_config->get('httpauth.method');

core/scripts/run-tests.sh

@@ -69,7 +69,9 @@
 simpletest_script_reporter_init();
 
 // Execute tests.
-simpletest_script_execute_batch(simpletest_script_get_test_list());
+for($test_run = 0; $test_run < $args['repeat']; $test_run++) {
+  simpletest_script_execute_batch(simpletest_script_get_test_list());
+}
 
 // Stop the timer.
 simpletest_script_reporter_timer_stop();
@@ -146,6 +148,14 @@ function simpletest_script_help() {
               Keeps detailed assertion results (in the database) after tests
               have completed. By default, assertion results are cleared.
 
+  --repeat    Number of times to repeat the test.
+
+  --die-on-fail
+
+              Exit out of php execution immediately on fail. This allows user to
+              change settings.php to use simpletest created databases and debug
+              failure. This is often used with repeat to find random failures.
+
   <test1>[,<test2>[,<test3> ...]]
 
               One or more tests to be run. By default, these are interpreted
@@ -191,6 +201,8 @@ function simpletest_script_parse_args() {
     'verbose' => FALSE,
     'keep-results' => FALSE,
     'test_names' => array(),
+    'repeat' => 1,
+    'die-on-fail' => FALSE,
     // Used internally.
     'test-id' => 0,
     'execute-test' => '',
@@ -200,6 +212,9 @@ function simpletest_script_parse_args() {
   // Override with set values.
   $args['script'] = basename(array_shift($_SERVER['argv']));
 
+  // Provide a default for repeat argument.
+  $args['repeat'] = 1;
+
   $count = 0;
   while ($arg = array_shift($_SERVER['argv'])) {
     if (preg_match('/--(\S+)/', $arg, $matches)) {
@@ -348,6 +363,13 @@ function simpletest_script_execute_batch($test_classes) {
         proc_close($child['process']);
         if ($status['exitcode']) {
           echo 'FATAL ' . $child['class'] . ': test runner returned a non-zero error code (' . $status['exitcode'] . ').' . "\n";
+          if ($args['die-on-fail']) {
+            list($db_prefix, ) = simpletest_last_test_get($child['test_id']);
+            $public_files = variable_get('file_public_path', conf_path() . '/files');
+            $test_directory = $public_files . '/simpletest/' . substr($db_prefix, 10);
+            echo 'Simpletest database and files kept and test exited immediately on fail so should be reproducible if you change settings.php to use the database prefix '. $db_prefix . ' and config directories in '. $test_directory . "\n";
+            exit();
+          }
         }
         // Free-up space by removing any potentially created resources.
         if (!$args['keep-results']) {
@@ -376,6 +398,7 @@ function simpletest_script_run_one_test($test_id, $test_class) {
     // Override configuration according to command line parameters.
     $conf['simpletest.settings']['verbose'] = $args['verbose'];
     $conf['simpletest.settings']['clear_results'] = !$args['keep-results'];
+    $conf['simpletest.settings']['die_on_fail'] = $args['die-on-fail'];
 
     $test = new $test_class($test_id);
     $test->run();
@@ -412,7 +435,7 @@ function simpletest_script_command($test_id, $test_class) {
   $command .= ' --url ' . escapeshellarg($args['url']);
   $command .= ' --php ' . escapeshellarg($php);
   $command .= " --test-id $test_id";
-  foreach (array('verbose', 'keep-results', 'color') as $arg) {
+  foreach (array('verbose', 'keep-results', 'color', 'die-on-fail') as $arg) {
     if ($args[$arg]) {
       $command .= ' --' . $arg;
     }
@@ -534,7 +557,9 @@ function simpletest_script_get_test_list() {
           'key' => 'name',
           'recurse' => TRUE,
         ));
-        $test_list = array_merge($test_list, array_keys($files));
+        foreach ($files as $test => $file) {
+          $test_list[] = "Drupal\\$module\\Tests\\$test";
+        }
       }
     }
     elseif ($args['file']) {