Issue #2869825 by justafish, dawehner, alexpott, martin107, jibran, Lendude,...

Issue #2869825 by justafish, dawehner, alexpott, martin107, jibran, Lendude, Mixologic, michielnugter, droplet, drpal, lauriii, effulgentsia, mglaman: Leverage JS for JS testing (using nightwatch)

core/.env.example

+# This is a dotenv file used by JavaScript tasks.
+# Copy this to '.env' to override.
+
+#############################
+# General Test Environment #
+#############################
+# This is the URL that Drupal can be accessed by. You don't need an installed
+# site here, just make sure you can at least access the installer screen. If you
+# don't already have one running, e.g. Apache, you can use PHP's built-in web
+# server by running the following command in your Drupal root folder:
+# php -S localhost:8888 .ht.router.php
+# DRUPAL_TEST_BASE_URL=http://localhost:8888
+DRUPAL_TEST_BASE_URL=
+
+# Tests need to be executed with a user in the same group as the web server
+# user.
+#DRUPAL_TEST_WEBSERVER_USER=www-data
+
+# By default we use sqlite as database. Use
+# mysql://username:password@localhost/databasename#table_prefix for mysql.
+DRUPAL_TEST_DB_URL=sqlite://localhost/sites/default/files/db.sqlite
+
+#############
+# Webdriver #
+#############
+
+# If Chromedriver is running as a service elsewhere, set it here.
+# When using DRUPAL_TEST_CHROMEDRIVER_AUTOSTART leave this at the default settings.
+DRUPAL_TEST_WEBDRIVER_HOSTNAME=localhost
+DRUPAL_TEST_WEBDRIVER_PORT=9515
+
+# If using Selenium, override the path prefix here.
+# See http://nightwatchjs.org/gettingstarted#browser-drivers-setup
+#DRUPAL_TEST_WEBDRIVER_PATH_PREFIX=/wd/hub
+
+################
+# Chromedriver #
+################
+
+# Automatically start chromedriver for local development. Set to false when you
+# use your own webdriver or chromedriver setup.
+# Also set it to false when you use a different browser for testing.
+DRUPAL_TEST_CHROMEDRIVER_AUTOSTART=true
+
+# A list of arguments to pass to Chrome, separated by spaces
+# e.g. `--disable-gpu --headless --no-sandbox`.
+#DRUPAL_TEST_WEBDRIVER_CHROME_ARGS=
+
+##############
+# Nightwatch #
+##############
+
+# Nightwatch generates output files. Use this to specify the location where these
+# files need to be stored. The default location is ignored by git, if you modify
+# the location you will probably want to add this location to your .gitignore.
+DRUPAL_NIGHTWATCH_OUTPUT=reports/nightwatch
+
+# The path that Nightwatch searches for assumes the same directory structure as
+# when you download Drupal core. If you have Drupal installed into a docroot
+# folder, you can use the following folder structure to add integration tests
+# for your project, outside of tests specifically for custom modules/themes/profiles.
+#
+# .
+# ├── docroot
+# │   ├── core
+# ├── tests
+# │   ├── Nightwatch
+# │   │   ├── Tests
+# │   │   │   ├── myTest.js
+#
+# and then set DRUPAL_NIGHTWATCH_SEARCH_DIRECTORY=../
+#
+#DRUPAL_NIGHTWATCH_SEARCH_DIRECTORY=
+
+# Filter directories to look for tests. This uses minimatch syntax.
+# Separate folders with a comma.
+DRUPAL_NIGHTWATCH_IGNORE_DIRECTORIES=node_modules,vendor,.*,sites/*/files,sites/*/private,sites/simpletest

core/.eslintignore

@@ -4,3 +4,5 @@ node_modules/**/*
 *.js
 !*.es6.js
 modules/locale/tests/locale_test.es6.js
+!nightwatch.conf.js
+!tests/Drupal/Nightwatch/**/*.js

core/.gitignore

 # Ignore node_modules folder created when installing core's JavaScript
 # dependencies.
 node_modules
+yarn-error.log
 
 # Ignore overrides of core's phpcs.xml.dist and phpunit.xml.dist.
 phpcs.xml
@@ -9,3 +10,12 @@ phpunit.xml
 # Ignore package-lock.json that is automatically created when adding
 # dependencies by users of NPMv5.
 package-lock.json
+
+# Ignore test reports
+reports
+
+# Ignore local Nightwatch settings
+nightwatch.settings.json
+
+# Ignore dotenv
+.env

core/package.json

@@ -3,16 +3,21 @@
   "description": "Drupal is an open source content management platform powering millions of websites and applications.",
   "license": "GPL-2.0",
   "private": true,
+  "engines": {
+    "yarn": ">= 1.6",
+    "node": ">= 8.11"
+  },
   "scripts": {
-    "build:js": "node ./scripts/js/babel-es6-build.js",
-    "build:js-dev": "cross-env NODE_ENV=development node ./scripts/js/babel-es6-build.js",
-    "watch:js": "node ./scripts/js/babel-es6-watch.js",
-    "watch:js-dev": "cross-env NODE_ENV=development node ./scripts/js/babel-es6-watch.js",
+    "build:js": "cross-env BABEL_ENV=legacy node ./scripts/js/babel-es6-build.js",
+    "build:js-dev": "cross-env NODE_ENV=development node BABEL_ENV=legacy ./scripts/js/babel-es6-build.js",
+    "watch:js": "cross-env BABEL_ENV=legacy node ./scripts/js/babel-es6-watch.js",
+    "watch:js-dev": "cross-env NODE_ENV=development BABEL_ENV=legacy node ./scripts/js/babel-es6-watch.js",
     "lint:core-js": "node ./node_modules/eslint/bin/eslint.js --ext=.es6.js . || exit 0",
     "lint:core-js-passing": "node ./node_modules/eslint/bin/eslint.js --quiet --config=.eslintrc.passing.json --ext=.es6.js . || exit 0",
     "lint:core-js-stats": "node ./node_modules/eslint/bin/eslint.js --format=./scripts/js/eslint-stats-by-type.js --ext=.es6.js . || exit 0",
     "lint:css": "stylelint \"**/*.css\" || exit 0",
-    "lint:css-checkstyle": "stylelint \"**/*.css\" --custom-formatter ./node_modules/stylelint-checkstyle-formatter/index.js || exit 0"
+    "lint:css-checkstyle": "stylelint \"**/*.css\" --custom-formatter ./node_modules/stylelint-checkstyle-formatter/index.js || exit 0",
+    "test:nightwatch": "cross-env BABEL_ENV=development node -r dotenv-safe/config -r babel-register ./node_modules/.bin/nightwatch --config ./tests/Drupal/Nightwatch/nightwatch.conf.js"
   },
   "devDependencies": {
     "babel-core": "^6.26.0",
@@ -20,37 +25,59 @@
     "babel-preset-env": "^1.4.0",
     "chalk": "^2.3.0",
     "chokidar": "^2.0.0",
+    "chromedriver": "^2.35.0",
     "cross-env": "^5.1.3",
+    "dotenv-safe": "^5.0.1",
     "eslint": "^3.19.0",
     "eslint-config-airbnb": "^14.1.0",
     "eslint-plugin-import": "^2.2.0",
     "eslint-plugin-jsx-a11y": "^4.0.0",
     "eslint-plugin-react": "^6.10.3",
-    "glob": "^7.1.1",
+    "glob": "^7.1.2",
     "minimist": "^1.2.0",
+    "mkdirp": "^0.5.1",
+    "nightwatch": "^0.9.20",
     "stylelint": "^9.1.1",
     "stylelint-checkstyle-formatter": "^0.1.1",
     "stylelint-config-standard": "^18.2.0",
     "stylelint-no-browser-hacks": "^1.1.0"
   },
+  "//": "'development is the default environment, and legacy is for transpiling the old jQuery codebase",
   "babel": {
-    "presets": [
-      [
-        "env",
-        {
-          "modules": false,
-          "targets": {
-            "browsers": [
-              "ie >= 11",
-              "edge >= 13",
-              "firefox >= 5",
-              "opera >= 12",
-              "safari >= 5",
-              "chrome >= 56"
-            ]
-          }
-        }
-      ]
-    ]
+    "env": {
+      "development": {
+        "presets": [
+          [
+            "env",
+            {
+              "modules": "commonjs",
+              "targets": {
+                "node": "current"
+              }
+            }
+          ]
+        ]
+      },
+      "legacy": {
+        "presets": [
+          [
+            "env",
+            {
+              "modules": false,
+              "targets": {
+                "browsers": [
+                  "ie >= 9",
+                  "edge >= 13",
+                  "firefox >= 5",
+                  "opera >= 12",
+                  "safari >= 5",
+                  "chrome >= 56"
+                ]
+              }
+            }
+          ]
+        ]
+      }
+    }
   }
 }

core/tests/Drupal/Nightwatch/Commands/installDrupal.js

+import { execSync } from 'child_process';
+import { URL } from 'url';
+import { commandAsWebserver } from '../globals';
+
+/**
+ * Installs a Drupal test site.
+ *
+ * @param {Object}
+ *   (optional) Settings object
+ *   @param setupFile
+ *     (optional) Setup file used by TestSiteApplicationTest
+ * @param {function} callback
+ *   A callback which will be called, when the installation is finished.
+ * @return {object}
+ *   The 'browser' object.
+ */
+exports.command = function installDrupal({ setupFile = '' }, callback) {
+  const self = this;
+
+  try {
+    setupFile = setupFile ? `--setup-file "${setupFile}"` : '';
+    const dbOption = process.env.DRUPAL_TEST_DB_URL.length > 0 ? `--db-url ${process.env.DRUPAL_TEST_DB_URL}` : '';
+    const install = execSync(commandAsWebserver(`php ./scripts/test-site.php install ${setupFile} --base-url ${process.env.DRUPAL_TEST_BASE_URL} ${dbOption} --json`));
+    const installData = JSON.parse(install.toString());
+    this.drupalDbPrefix = installData.db_prefix;
+    const url = new URL(process.env.DRUPAL_TEST_BASE_URL);
+    this
+      .url(process.env.DRUPAL_TEST_BASE_URL)
+      .setCookie({
+        name: 'SIMPLETEST_USER_AGENT',
+        // Colons need to be URL encoded to be valid.
+        value: encodeURIComponent(installData.user_agent),
+        path: url.pathname,
+        domain: url.host,
+      });
+  }
+  catch (error) {
+    this.assert.fail(error);
+  }
+
+  // Nightwatch doesn't like it when no actions are added in a command file.
+  // https://github.com/nightwatchjs/nightwatch/issues/1792
+  this.pause(1);
+
+  if (typeof callback === 'function') {
+    callback.call(self);
+  }
+  return this;
+};

core/tests/Drupal/Nightwatch/Commands/logAndEnd.js

+/**
+ * Ends the browser session and logs the console log if there were any errors.
+ * See globals.js.
+ *
+ * @param {Object}
+ *   (optional) Settings object
+ *   @param onlyOnError
+ *     (optional) Only writes out the console log file if the test failed.
+ * @param {function} callback
+ *   A callback which will be called.
+ * @return {object}
+ *   The 'browser' object.
+ */
+exports.command = function logAndEnd({ onlyOnError = true }, callback) {
+  const self = this;
+  this.drupalLogConsole = true;
+  this.drupalLogConsoleOnlyOnError = onlyOnError;
+
+  // Nightwatch doesn't like it when no actions are added in a command file.
+  // https://github.com/nightwatchjs/nightwatch/issues/1792
+  this.pause(1);
+
+  if (typeof callback === 'function') {
+    callback.call(self);
+  }
+  return this;
+};

core/tests/Drupal/Nightwatch/Commands/relativeURL.js

+/**
+ * Concatenate a DRUPAL_TEST_BASE_URL variable and a pathname.
+ *
+ * This provides a custom command, .relativeURL()
+ *
+ * @param {string} pathname
+ *   The relative path to append to DRUPAL_TEST_BASE_URL
+ * @param {function} callback
+ *   A callback which will be called.
+ * @return {object}
+ *   The 'browser' object.
+ */
+exports.command = function relativeURL(pathname, callback) {
+  const self = this;
+  this.url(`${process.env.DRUPAL_TEST_BASE_URL}${pathname}`);
+
+  if (typeof callback === 'function') {
+    callback.call(self);
+  }
+  return this;
+};

core/tests/Drupal/Nightwatch/Commands/uninstallDrupal.js

+import { execSync } from 'child_process';
+import { commandAsWebserver } from '../globals';
+
+/**
+ * Uninstalls a test Drupal site.
+ *
+ * @param {function} callback
+ *   A callback which will be called, when the uninstallation is finished.
+ * @return {object}
+ *   The 'browser' object.
+ */
+exports.command = function uninstallDrupal(callback) {
+  const self = this;
+  const prefix = self.drupalDbPrefix;
+
+  // Check for any existing errors, because running this will cause Nightwatch to hang.
+  if (!this.currentTest.results.errors && !this.currentTest.results.failed) {
+    const dbOption = process.env.DRUPAL_TEST_DB_URL.length > 0 ? `--db-url ${process.env.DRUPAL_TEST_DB_URL}` : '';
+    try {
+      if (!prefix || !prefix.length) {
+        throw new Error('Missing database prefix parameter, unable to uninstall Drupal (the initial install was probably unsuccessful).');
+      }
+      execSync(commandAsWebserver(`php ./scripts/test-site.php tear-down ${prefix} ${dbOption}`));
+    }
+    catch (error) {
+      this.assert.fail(error);
+    }
+  }
+
+  // Nightwatch doesn't like it when no actions are added in a command file.
+  // https://github.com/nightwatchjs/nightwatch/issues/1792
+  this.pause(1);
+
+  if (typeof callback === 'function') {
+    callback.call(self);
+  }
+  return this;
+};

core/tests/Drupal/Nightwatch/Tests/exampleTest.js

+module.exports = {
+  '@tags': ['core'],
+  before(browser) {
+    browser
+      .installDrupal({ setupFile: 'core/tests/Drupal/TestSite/TestSiteInstallTestScript.php' });
+  },
+  after(browser) {
+    browser
+      .uninstallDrupal();
+  },
+  'Test page': (browser) => {
+    browser
+      .relativeURL('/test-page')
+      .waitForElementVisible('body', 1000)
+      .assert.containsText('body', 'Test page text')
+      .logAndEnd({ onlyOnError: false });
+  },
+  /**
+  'Example failing test': (browser) => {
+    browser
+      // Change this to point at a site which has some console errors, as the
+      // test site that was just installed doesn't.
+      .url('https://www./')
+      .waitForElementVisible('h1', 1000)
+      // Wait for some errors to build up.
+      .pause(5000)
+      .assert.containsText('h1', 'I\'m the operator with my pocket calculator')
+      .logAndEnd();
+  },
+  **/
+};

core/tests/Drupal/Nightwatch/globals.js

+import { spawn } from 'child_process';
+import path from 'path';
+import fs from 'fs';
+import mkdirp from 'mkdirp';
+import chromedriver from 'chromedriver';
+import nightwatchSettings from './nightwatch.conf';
+
+const commandAsWebserver = (command) => {
+  if (process.env.DRUPAL_TEST_WEBSERVER_USER) {
+    return `sudo -u ${process.env.DRUPAL_TEST_WEBSERVER_USER} ${command}`;
+  }
+  return command;
+};
+
+module.exports = {
+  before: (done) => {
+    if (JSON.parse(process.env.DRUPAL_TEST_CHROMEDRIVER_AUTOSTART)) {
+      chromedriver.start();
+    }
+    done();
+  },
+  after: (done) => {
+    if (JSON.parse(process.env.DRUPAL_TEST_CHROMEDRIVER_AUTOSTART)) {
+      chromedriver.stop();
+    }
+    done();
+  },
+  afterEach: (browser, done) => {
+    // Writes the console log - used by the "logAndEnd" command.
+    if (
+      browser.drupalLogConsole &&
+      (!browser.drupalLogConsoleOnlyOnError || browser.currentTest.results.errors > 0 || browser.currentTest.results.failed > 0)
+    ) {
+      let testName = browser.currentTest.name || browser.currentTest.module;
+      testName = testName.split(' ').join('-');
+      const resultPath = path.join(__dirname, `../../../${nightwatchSettings.output_folder}/consoleLogs/${browser.currentTest.module}`);
+      const status = browser.currentTest.results.errors > 0 || browser.currentTest.results.failed > 0 ? 'FAILED' : 'PASSED';
+      mkdirp.sync(resultPath);
+      const now = new Date().toString().split(' ').join('-');
+      browser
+        .getLog('browser', (logEntries) => {
+          const browserLog = JSON.stringify(logEntries, null, '  ');
+          fs.writeFileSync(`${resultPath}/${testName}_${status}_${now}_console.json`, browserLog);
+        })
+        .end(done);
+    }
+    else {
+      browser.end(done);
+    }
+  },
+  commandAsWebserver,
+};

core/tests/Drupal/Nightwatch/nightwatch.conf.js

+import path from 'path';
+import glob from 'glob';
+
+// Find directories which have Nightwatch tests in them.
+const regex = /(.*\/?tests\/?.*\/Nightwatch)\/.*/g;
+const collectedFolders = {
+  Tests: [],
+  Commands: [],
+  Assertions: [],
+};
+const searchDirectory = process.env.DRUPAL_NIGHTWATCH_SEARCH_DIRECTORY || '';
+
+glob
+  .sync('**/tests/**/Nightwatch/**/*.js', {
+    cwd: path.resolve(process.cwd(), `../${searchDirectory}`),
+    ignore: process.env.DRUPAL_NIGHTWATCH_IGNORE_DIRECTORIES ?
+      process.env.DRUPAL_NIGHTWATCH_IGNORE_DIRECTORIES.split(',') : [],
+  })
+  .forEach((file) => {
+    let m = regex.exec(file);
+    while (m !== null) {
+      // This is necessary to avoid infinite loops with zero-width matches.
+      if (m.index === regex.lastIndex) {
+        regex.lastIndex += 1;
+      }
+
+      const key = `../${m[1]}`;
+      Object.keys(collectedFolders).forEach((folder) => {
+        if (file.includes(`Nightwatch/${folder}`)) {
+          collectedFolders[folder].push(`${searchDirectory}${key}/${folder}`);
+        }
+      });
+      m = regex.exec(file);
+    }
+  });
+
+// Remove duplicate folders.
+Object.keys(collectedFolders).forEach((folder) => {
+  collectedFolders[folder] = Array.from(new Set(collectedFolders[folder]));
+});
+
+module.exports = {
+  src_folders: collectedFolders.Tests,
+  output_folder: process.env.DRUPAL_NIGHTWATCH_OUTPUT,
+  custom_commands_path: collectedFolders.Commands,
+  custom_assertions_path: collectedFolders.Assertions,
+  page_objects_path: '',
+  globals_path: 'tests/Drupal/Nightwatch/globals.js',
+  selenium: {
+    start_process: false,
+  },
+  test_settings: {
+    default: {
+      selenium_port: process.env.DRUPAL_TEST_WEBDRIVER_PORT,
+      selenium_host: process.env.DRUPAL_TEST_WEBDRIVER_HOSTNAME,
+      default_path_prefix: process.env.DRUPAL_TEST_WEBDRIVER_PATH_PREFIX || '',
+      desiredCapabilities: {
+        browserName: 'chrome',
+        acceptSslCerts: true,
+        chromeOptions: {
+          args: process.env.DRUPAL_TEST_WEBDRIVER_CHROME_ARGS ? process.env.DRUPAL_TEST_WEBDRIVER_CHROME_ARGS.split(' ') : [],
+        },
+      },
+      screenshots: {
+        enabled: true,
+        on_failure: true,
+        on_error: true,
+        path: `${process.env.DRUPAL_NIGHTWATCH_OUTPUT}/screenshots`,
+      },
+      end_session_on_fail: false,
+    },
+  },
+};

core/tests/README.md

@@ -102,3 +102,34 @@ export SIMPLETEST_BASE_URL='http://d8.dev'
 sudo -u www-data -E ./vendor/bin/phpunit -c core --testsuite functional
 sudo -u www-data -E ./vendor/bin/phpunit -c core --testsuite functional-javascript
 ```
+
+## Nightwatch tests
+
+- Ensure your vendor directory is populated (e.g. by running `composer install`)
+- If you're running PHP 7.0 or greater you will need to upgrade PHPUnit with `composer run-script drupal-phpunit-upgrade`
+- Install [Node.js](https://nodejs.org/en/download/) and [yarn](https://yarnpkg.com/en/docs/install). The versions required are specificed inside core/package.json in the `engines` field
+- Install [Google Chrome](https://www.google.com/chrome/browser/desktop/index.html)
+- Inside the `core` folder, run `yarn install`
+- Configure the nightwatch settings by copying `.env.example` to `.env` and editing as necessary.
+- Ensure you have a web server running (as instructed in `.env`)
+- Again inside the `core` folder, run `yarn test:nightwatch` to run the tests. By default this will output reports to `core/reports`
+- Nightwatch will run tests for core, as well as contrib and custom modules and themes. It will search for tests located under folders with the pattern `**/tests/**/Nightwatch/(Tests|Commands|Assertions)`
+- To run only core tests, run `yarn test:nightwatch --tag core`
+- To skip running core tests, run `yarn test:nightwatch --skiptags core`
+- To run a single test, run e.g. `yarn test:nightwatch tests/Drupal/Nightwatch/Tests/exampleTest.js`
+
+Nightwatch tests can be placed in any folder with the pattern `**/tests/**/Nightwatch/(Tests|Commands|Assertions)`. For example:
+```
+tests/Nightwatch/Tests
+src/tests/Nightwatch/Tests
+tests/src/Nightwatch/Tests
+tests/Nightwatch/Commands
+```
+
+It's helpful to follow existing patterns for test placement, so for the action module they would go in `core/modules/action/tests/src/Nightwatch`.
+The Nightwatch configuration, as well as global tests, commands, and assertions which span many modules/systems, are located in `core/tests/Drupal/Nightwatch`.
+
+If your core directory is located in a subfolder (e.g. `docroot`), then you can edit the search directory in `.env` to pick up tests outside of your Drupal directory.
+Tests outside of the `core` folder will run in the version of node you have installed. If you want to transpile with babel (e.g. to use `import` statements) outside of core,
+then add your own babel config to the root of your project. For example, if core is located under `docroot/core`, then you could run `yarn add babel-preset-env` inside
+`docroot`, then copy the babel settings from `docroot/core/package.json` into `docroot/package.json`.