Implement e2e functional smoke testing and visual regression testing with playwright (#4456)

* First example of playwright

* Move config

* set junit output

* fix config and update version

* staged changes

* new files

* update to remote dir

* remove remote

* smoke

* Update smoke.spec.js

* Add eslint for playwright

* Add dependabot rule for playwright

* Add adhoc GHA for playwright

* Update tests and fix eslint

* move playwright eslint config to e2e dir

* Add to circle config

* store artifacts

* wrong location for storing artifacts

* slash?

* build before start

* increase timeouts in circle

* remove duplicate build step

* timeout values

* add retries

* reduce retries

* add percy emblem

* added percy

* Adds GHA for adhoc trigger and baseline visual

* Bump and Rev dependabot separte from karma

* update e2e label for additional pr logic

* Ignore playwright changes for codeql

* Update documentation

* Add ability to run all tests on pr label

* eof

* yamlamlaml

* issue instead of pull api

* archive results and include a link in the PR comment

* log context and attempt string concat

* concat fix

* add success/failure options

* add alure reporting

* lock playwright image

Co-authored-by: unlikelyzero <jchill2@gmail.com>

e2e/.eslintrc.js

@@ -0,0 +1,4 @@
+/* eslint-disable no-undef */
+module.exports = {
+    "extends": ["plugin:playwright/playwright-test"]
+};

e2e/playwright-ci.config.js

@@ -0,0 +1,32 @@
+/* eslint-disable no-undef */
+// playwright.config.js
+// @ts-check
+
+/** @type {import('@playwright/test').PlaywrightTestConfig} */
+const config = {
+    retries: 2,
+    testDir: 'tests',
+    timeout: 90 * 1000,
+    webServer: {
+        command: 'npm run start',
+        port: 8080,
+        timeout: 200 * 1000,
+        reuseExistingServer: !process.env.CI
+    },
+    use: {
+        browserName: "chromium",
+        baseURL: 'http://localhost:8080/',
+        headless: true,
+        ignoreHTTPSErrors: true,
+        screenshot: 'on',
+        trace: 'on',
+        video: 'on'
+    },
+    reporter: [
+        ['list'],
+        ['junit', { outputFile: 'test-results/results.xml' }],
+        ['allure-playwright']
+    ]
+};
+
+module.exports = config;

e2e/playwright-local.config.js

@@ -0,0 +1,31 @@
+/* eslint-disable no-undef */
+// playwright.config.js
+// @ts-check
+
+/** @type {import('@playwright/test').PlaywrightTestConfig} */
+const config = {
+    retries: 0,
+    testDir: 'tests',
+    timeout: 30 * 1000,
+    webServer: {
+        command: 'npm run start',
+        port: 8080,
+        timeout: 120 * 1000,
+        reuseExistingServer: !process.env.CI
+    },
+    use: {
+        browserName: "chromium",
+        baseURL: 'http://localhost:8080/',
+        headless: false,
+        ignoreHTTPSErrors: true,
+        screenshot: 'on',
+        trace: 'on',
+        video: 'on'
+    },
+    reporter: [
+        ['list'],
+        ['allure-playwright']
+    ]
+};
+
+module.exports = config;

e2e/playwright-visual.config.js

@@ -0,0 +1,32 @@
+/* eslint-disable no-undef */
+// playwright.config.js
+// @ts-check
+
+/** @type {import('@playwright/test').PlaywrightTestConfig} */
+const config = {
+    retries: 0,
+    testDir: 'tests',
+    timeout: 90 * 1000,
+    webServer: {
+        command: 'npm run start',
+        port: 8080,
+        timeout: 200 * 1000,
+        reuseExistingServer: !process.env.CI
+    },
+    use: {
+        browserName: "chromium",
+        baseURL: 'http://localhost:8080/',
+        headless: true,
+        ignoreHTTPSErrors: true,
+        screenshot: 'on',
+        trace: 'off',
+        video: 'on'
+    },
+    reporter: [
+        ['list'],
+        ['junit', { outputFile: 'test-results/results.xml' }],
+        ['allure-playwright']
+    ]
+};
+
+module.exports = config;

e2e/tests/smoke.spec.js

@@ -0,0 +1,49 @@
+/*****************************************************************************
+ * Open MCT, Copyright (c) 2014-2021, United States Government
+ * as represented by the Administrator of the National Aeronautics and Space
+ * Administration. All rights reserved.
+ *
+ * Open MCT is licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * Open MCT includes source code licensed under additional open source
+ * licenses. See the Open Source Licenses file (LICENSES.md) included with
+ * this source code distribution or the Licensing information page available
+ * at runtime from the About dialog for additional information.
+ *****************************************************************************/
+
+/*
+This test suite is dedicated to tests which can quickly verify that any openmct installation is
+operable and that any type of testing can proceed.
+
+Ideally, smoke tests should make zero assumptions about how and where they are run. This makes them
+more resilient to change and therefor a better indicator of failure. Smoke tests will also run quickly
+as they cover a very "thin surface" of functionality.
+
+When deciding between authoring new smoke tests or functional tests, ask yourself "would I feel
+comfortable running this test during a live mission?" Avoid creating or deleting Domain Objects.
+Make no assumptions about the order that elements appear in the DOM.
+*/
+
+const { test, expect } = require('@playwright/test');
+
+test('Verify that the create button appears and that the Folder Domain Object is available for selection', async ({ page }) => {
+
+    //Go to baseURL
+    await page.goto('/', { waitUntil: 'networkidle' });
+
+    //Click the Create button
+    await page.click('button:has-text("Create")');
+
+    // Verify that Create Folder appears in the dropdown
+    const locator = page.locator(':nth-match(:text("Folder"), 2)');
+    await expect(locator).toBeEnabled();
+});

e2e/tests/visual/default.spec.js

@@ -0,0 +1,73 @@
+/*****************************************************************************
+ * Open MCT, Copyright (c) 2014-2021, United States Government
+ * as represented by the Administrator of the National Aeronautics and Space
+ * Administration. All rights reserved.
+ *
+ * Open MCT is licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * Open MCT includes source code licensed under additional open source
+ * licenses. See the Open Source Licenses file (LICENSES.md) included with
+ * this source code distribution or the Licensing information page available
+ * at runtime from the About dialog for additional information.
+ *****************************************************************************/
+
+/*
+Collection of Visual Tests set to run in a default context. These should only use functional
+expect statements to verify assumptions about the state in a test and not for functional
+verification of correctness.
+Note: Larger testsuite sizes are OK due to the setup time associated with these tests. Visual
+tests are not supposed to "fail" on assertions.
+*/
+
+const { test, expect } = require('@playwright/test');
+const percySnapshot = require('@percy/playwright');
+const path = require('path');
+const sinon = require('sinon');
+
+const VISUAL_GRACE_PERIOD = 5 * 1000; //Lets the application "simmer" before the snapshot is taken
+
+// Snippet from https://github.com/microsoft/playwright/issues/6347#issuecomment-965887758
+// Will replace with cy.clock() equivalent
+test.beforeEach(async ({ context }) => {
+    await context.addInitScript({
+        // eslint-disable-next-line no-undef
+        path: path.join(__dirname, '../../..', './node_modules/sinon/pkg/sinon.js')
+    });
+    await context.addInitScript(() => {
+        window.__clock = sinon.useFakeTimers(); //Set browser clock to UNIX Epoch
+    });
+});
+
+test('Visual - Root and About', async ({ page }) => {
+    // Go to baseURL
+    await page.goto('/', { waitUntil: 'networkidle' });
+
+    // Verify that Create button is actionable
+    const createButtonLocator = page.locator('button:has-text("Create")');
+    await expect(createButtonLocator).toBeEnabled();
+
+    // Take a snapshot of the Dashboard
+    await page.waitForTimeout(VISUAL_GRACE_PERIOD);
+    await percySnapshot(page, 'Root');
+
+    // Click About button
+    await page.click('.l-shell__app-logo');
+
+    // Modify the Build information in 'about' to be consistent run-over-run
+    const versionInformationLocator = page.locator('ul.t-info.l-info.s-info');
+    await expect(versionInformationLocator).toBeEnabled();
+    await versionInformationLocator.evaluate(node => node.innerHTML = '<li>Version: visual-snapshot</li> <li>Build Date: Mon Nov 15 2021 08:07:51 GMT-0800 (Pacific Standard Time)</li> <li>Revision: 93049cdbc6c047697ca204893db9603b864b8c9f</li> <li>Branch: master</li>');
+
+    // Take a snapshot of the About modal
+    await page.waitForTimeout(VISUAL_GRACE_PERIOD);
+    await percySnapshot(page, 'About');
+});