[CI] Add a11y checks to our visual testing suite (#7047)

* Add a VISUAL_URL constant and remove all vestiges of hide inspector and tree

* hide timer and add concurrency

* turn off concurrency

* factor out old appAction

* Add expand button to panes

* remove old slow annotations

* fix fault

* update domcontentloaded

* missed refactor

* driveby: setTimeBounds private

* add comments to the percyCSS

* suggest MISSION_TIME

* more notes

* regen

* clean up test

* driveby: clean up order

* restructure

* add new suite now that i'ts hidden

* use mission time everywhere possible

* driveby

* rerun generatedata

* comments

* lint fix

* add inital pass of a11y tests

* first pass for fixing a11y problems

* update build

* add copyright

* check for slashes

* rename files

* update testcases

* update to latest

* updates

* section 508

* final version

* remove leftover

* comments

* documentation

* bad merge

* comment

* use current ruleset

* typo

* feedback

* remove time conductor due to false positives

* default to closed tabs

* add some more accessiblity checking

* change to a condition widget and update search

* lint fix

* turns this into a single function

* update doc to match single function

* update to single function

* update to new function

* lint

* update locator for search input

* fix extra page

* why

* comments

* comments

* refacotr

* wrong paths and fixes
This commit is contained in:
John Hill
2023-12-19 14:16:08 -08:00
committed by GitHub
parent ec910dcbdc
commit 0d97675a0a
31 changed files with 480 additions and 381 deletions

View File

@@ -25,6 +25,11 @@ Tests to verify plot tagging functionality.
*/
const { test, expect } = require('../../../../pluginFixtures');
const {
basicTagsTests,
createTags,
testTelemetryItem
} = require('../../../../helper/plotTagsUtils');
const {
createDomainObjectWithDefaults,
setRealTimeMode,
@@ -33,163 +38,6 @@ const {
} = require('../../../../appActions');
test.describe('Plot Tagging', () => {
/**
* Given a canvas and a set of points, tags the points on the canvas.
* @param {import('@playwright/test').Page} page
* @param {HTMLCanvasElement} canvas a telemetry item with a plot
* @param {Number} xEnd a telemetry item with a plot
* @param {Number} yEnd a telemetry item with a plot
* @returns {Promise}
*/
async function createTags({ page, canvas, xEnd = 700, yEnd = 520 }) {
await canvas.hover({ trial: true });
//Alt+Shift Drag Start to select some points to tag
await page.keyboard.down('Alt');
await page.keyboard.down('Shift');
await canvas.dragTo(canvas, {
sourcePosition: {
x: 1,
y: 1
},
targetPosition: {
x: xEnd,
y: yEnd
}
});
//Alt Drag End
await page.keyboard.up('Alt');
await page.keyboard.up('Shift');
//Wait for canvas to stabilize.
await canvas.hover({ trial: true });
// add some tags
await page.getByText('Annotations').click();
await page.getByRole('button', { name: /Add Tag/ }).click();
await page.getByPlaceholder('Type to select tag').click();
await page.getByText('Driving').click();
await page.getByRole('button', { name: /Add Tag/ }).click();
await page.getByPlaceholder('Type to select tag').click();
await page.getByText('Science').click();
}
/**
* Given a telemetry item (e.g., a Sine Wave Generator) with a plot, tests that the plot can be tagged.
* @param {import('@playwright/test').Page} page
* @param {import('../../../../appActions').CreatedObjectInfo} telemetryItem a telemetry item with a plot
* @returns {Promise}
*/
async function testTelemetryItem(page, telemetryItem) {
// Check that telemetry item also received the tag
await page.goto(telemetryItem.url);
await expect(page.getByText('No tags to display for this item')).toBeVisible();
const canvas = page.locator('canvas').nth(1);
//Wait for canvas to stabilize.
await waitForPlotsToRender(page);
await expect(canvas).toBeInViewport();
await canvas.hover({ trial: true });
// click on the tagged plot point
await canvas.click({
position: {
x: 100,
y: 100
}
});
await expect(page.getByText('Science')).toBeVisible();
await expect(page.getByText('Driving')).toBeHidden();
}
/**
* Given a page, tests that tags are searchable, deletable, and persist across reloads.
* @param {import('@playwright/test').Page} page
* @returns {Promise}
*/
async function basicTagsTests(page) {
// Search for Driving
await page.locator('[aria-label="OpenMCT Search"] input[type="search"]').click();
// Clicking elsewhere should cause annotation selection to be cleared
await expect(page.getByText('No tags to display for this item')).toBeVisible();
await page.locator('[aria-label="OpenMCT Search"] input[type="search"]').fill('driv');
// click on the search result
await page
.getByRole('searchbox', { name: 'OpenMCT Search' })
.getByText(/Sine Wave/)
.first()
.click();
// Delete Driving
await page.hover('[aria-label="Tag"]:has-text("Driving")');
await page.locator('[aria-label="Remove tag Driving"]').click();
// Search for Science
await page.locator('[aria-label="OpenMCT Search"] input[type="search"]').click();
await page.locator('[aria-label="OpenMCT Search"] input[type="search"]').fill('sc');
await expect(page.locator('[aria-label="Search Result"]').nth(0)).toContainText('Science');
await expect(page.locator('[aria-label="Search Result"]').nth(0)).not.toContainText('Drilling');
// Search for Driving
await page.locator('[aria-label="OpenMCT Search"] input[type="search"]').click();
await page.locator('[aria-label="OpenMCT Search"] input[type="search"]').fill('driv');
await expect(page.getByText('No results found')).toBeVisible();
//Reload Page
await page.reload({ waitUntil: 'domcontentloaded' });
// wait for plots to load
await waitForPlotsToRender(page);
await expect(page.getByRole('tab', { name: 'Annotations' })).not.toHaveClass(/is-current/);
await page.getByRole('tab', { name: 'Annotations' }).click();
await expect(page.getByRole('tab', { name: 'Annotations' })).toHaveClass(/is-current/);
await expect(page.getByText('No tags to display for this item')).toBeVisible();
const canvas = page.locator('canvas').nth(1);
// click on the tagged plot point
await canvas.click({
position: {
x: 100,
y: 100
}
});
await expect(page.getByText('Science')).toBeVisible();
await expect(page.getByText('Driving')).toBeHidden();
// click elsewhere
await page.locator('body').click();
//click on tagged plot point again
await canvas.click({
position: {
x: 100,
y: 100
}
});
// Add driving tag again
await page.getByText('Annotations').click();
await page.getByRole('button', { name: /Add Tag/ }).click();
await page.getByPlaceholder('Type to select tag').click();
await page.getByText('Driving').click();
await expect(page.getByText('Science')).toBeVisible();
await expect(page.getByText('Driving')).toBeVisible();
// Delete Driving again
await page.hover('[aria-label="Tag"]:has-text("Driving")');
await page.locator('[aria-label="Remove tag Driving"]').click();
await expect(page.getByText('Science')).toBeVisible();
await expect(page.getByText('Driving')).toBeHidden();
}
test.beforeEach(async ({ page }) => {
await page.goto('./', { waitUntil: 'domcontentloaded' });
});
@@ -244,19 +92,17 @@ test.describe('Plot Tagging', () => {
// set to real time mode
await setRealTimeMode(page);
// Search for Science
await page.locator('[aria-label="OpenMCT Search"] input[type="search"]').click();
await page.locator('[aria-label="OpenMCT Search"] input[type="search"]').fill('sc');
// click on the search result
await page
.getByRole('searchbox', { name: 'OpenMCT Search' })
.getByText('Alpha Sine Wave')
.first()
.click();
// wait for plots to load
await expect(page.locator('.js-series-data-loaded')).toBeVisible();
// Search for Science Tag
await page.getByRole('searchbox', { name: 'Search Input' });
await page.getByRole('searchbox', { name: 'Search Input' }).fill('sc');
// Click on the search object result
await page.getByLabel('OpenMCT Search').getByText('Alpha Sine Wave').first().click();
await waitForPlotsToRender(page);
// expect plot to be paused
await expect(page.locator('[title="Resume displaying real-time data"]')).toBeVisible();
await expect(page.getByTitle('Resume displaying real-time data')).toBeVisible();
await setFixedTimeMode(page);
});

View File

@@ -25,6 +25,7 @@ Tests to verify plot tagging performance.
*/
const { test, expect } = require('../../pluginFixtures');
const { basicTagsTests, createTags, testTelemetryItem } = require('../../helper/plotTagsUtils');
const {
createDomainObjectWithDefaults,
setRealTimeMode,
@@ -33,135 +34,6 @@ const {
} = require('../../appActions');
test.describe('Plot Tagging Performance', () => {
/**
* Given a canvas and a set of points, tags the points on the canvas.
* @param {import('@playwright/test').Page} page
* @param {HTMLCanvasElement} canvas a telemetry item with a plot
* @param {Number} xEnd a telemetry item with a plot
* @param {Number} yEnd a telemetry item with a plot
* @returns {Promise}
*/
async function createTags({ page, canvas, xEnd = 700, yEnd = 520 }) {
await canvas.hover({ trial: true });
//Alt+Shift Drag Start to select some points to tag
await page.keyboard.down('Alt');
await page.keyboard.down('Shift');
await canvas.dragTo(canvas, {
sourcePosition: {
x: 1,
y: 1
},
targetPosition: {
x: xEnd,
y: yEnd
}
});
//Alt Drag End
await page.keyboard.up('Alt');
await page.keyboard.up('Shift');
//Wait for canvas to stabilize.
await canvas.hover({ trial: true });
// add some tags
await page.getByText('Annotations').click();
await page.getByRole('button', { name: /Add Tag/ }).click();
await page.getByPlaceholder('Type to select tag').click();
await page.getByText('Driving').click();
await page.getByRole('button', { name: /Add Tag/ }).click();
await page.getByPlaceholder('Type to select tag').click();
await page.getByText('Science').click();
}
/**
* Given a telemetry item (e.g., a Sine Wave Generator) with a plot, tests that the plot can be tagged.
* @param {import('@playwright/test').Page} page
* @param {import('../../../../appActions').CreatedObjectInfo} telemetryItem a telemetry item with a plot
* @returns {Promise}
*/
async function testTelemetryItem(page, telemetryItem) {
// Check that telemetry item also received the tag
await page.goto(telemetryItem.url);
await expect(page.getByText('No tags to display for this item')).toBeVisible();
const canvas = page.locator('canvas').nth(1);
//Wait for canvas to stabilize.
await canvas.hover({ trial: true });
// click on the tagged plot point
await canvas.click({
position: {
x: 100,
y: 100
}
});
await expect(page.getByText('Science')).toBeVisible();
await expect(page.getByText('Driving')).toBeHidden();
}
/**
* Given a page, tests that tags are searchable, deletable, and persist across reloads.
* @param {import('@playwright/test').Page} page
* @returns {Promise}
*/
async function basicTagsTests(page) {
// Search for Driving
await page.locator('[aria-label="OpenMCT Search"] input[type="search"]').click();
// Clicking elsewhere should cause annotation selection to be cleared
await expect(page.getByText('No tags to display for this item')).toBeVisible();
await page.locator('[aria-label="OpenMCT Search"] input[type="search"]').fill('driv');
// click on the search result
await page
.getByRole('searchbox', { name: 'OpenMCT Search' })
.getByText(/Sine Wave/)
.first()
.click();
// Delete Driving
await page.hover('[aria-label="Tag"]:has-text("Driving")');
await page.locator('[aria-label="Remove tag Driving"]').click();
// Search for Science
await page.locator('[aria-label="OpenMCT Search"] input[type="search"]').click();
await page.locator('[aria-label="OpenMCT Search"] input[type="search"]').fill('sc');
await expect(page.locator('[aria-label="Search Result"]').nth(0)).toContainText('Science');
await expect(page.locator('[aria-label="Search Result"]').nth(0)).not.toContainText('Drilling');
// Search for Driving
await page.locator('[aria-label="OpenMCT Search"] input[type="search"]').click();
await page.locator('[aria-label="OpenMCT Search"] input[type="search"]').fill('driv');
await expect(page.getByText('No results found')).toBeVisible();
//Reload Page
await page.reload({ waitUntil: 'domcontentloaded' });
// wait for plots to load
await waitForPlotsToRender(page);
await page.getByText('Annotations').click();
await expect(page.getByText('No tags to display for this item')).toBeVisible();
const canvas = page.locator('canvas').nth(1);
// click on the tagged plot point
await canvas.click({
position: {
x: 100,
y: 100
}
});
await expect(page.getByText('Science')).toBeVisible();
await expect(page.getByText('Driving')).toBeHidden();
}
test.beforeEach(async ({ page }) => {
await page.goto('./', { waitUntil: 'domcontentloaded' });
});
@@ -212,18 +84,15 @@ test.describe('Plot Tagging Performance', () => {
await setRealTimeMode(page);
// Search for Science
await page.locator('[aria-label="OpenMCT Search"] input[type="search"]').click();
await page.locator('[aria-label="OpenMCT Search"] input[type="search"]').fill('sc');
await page.getByRole('searchbox', { name: 'Search Input' });
await page.getByRole('searchbox', { name: 'Search Input' }).fill('sc');
// click on the search result
await page
.getByRole('searchbox', { name: 'OpenMCT Search' })
.getByText('Alpha Sine Wave')
.first()
.click();
// wait for plots to load
await expect(page.locator('.js-series-data-loaded')).toBeVisible();
await page.getByLabel('Search Result').getByText('Alpha Sine Wave').first().click();
await waitForPlotsToRender(page);
// expect plot to be paused
await expect(page.locator('[title="Resume displaying real-time data"]')).toBeVisible();
await expect(page.getByTitle('Resume displaying real-time data')).toBeVisible();
await setFixedTimeMode(page);
});

View File

@@ -0,0 +1,33 @@
/*****************************************************************************
* Open MCT, Copyright (c) 2014-2023, United States Government
* as represented by the Administrator of the National Aeronautics and Space
* Administration. All rights reserved.
*
* Open MCT is licensed under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0.
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*
* Open MCT includes source code licensed under additional open source
* licenses. See the Open Source Licenses file (LICENSES.md) included with
* this source code distribution or the Licensing information page available
* at runtime from the About dialog for additional information.
*****************************************************************************/
const { test, scanForA11yViolations } = require('../../avpFixtures');
const VISUAL_URL = require('../../constants').VISUAL_URL;
test.describe('a11y - Default @a11y', () => {
test.beforeEach(async ({ page }) => {
await page.goto(VISUAL_URL, { waitUntil: 'domcontentloaded' });
});
test('main view @a11y', async ({ page }, testInfo) => {
await scanForA11yViolations(page, testInfo.title);
});
});

View File

@@ -26,12 +26,12 @@ are only meant to run against openmct's app.js started by `npm run start` within
`./e2e/playwright-visual.config.js` file.
*/
const { test, expect } = require('../../pluginFixtures');
const { test, expect, scanForA11yViolations } = require('../../avpFixtures');
const percySnapshot = require('@percy/playwright');
const { createDomainObjectWithDefaults } = require('../../appActions');
const { VISUAL_URL } = require('../../constants');
test.describe('Visual - Default', () => {
test.describe('Visual - Default @a11y', () => {
test.beforeEach(async ({ page }) => {
await page.goto(VISUAL_URL, { waitUntil: 'domcontentloaded' });
});
@@ -98,4 +98,8 @@ test.describe('Visual - Default', () => {
// Take a snapshot of the newly created Gauge object
await percySnapshot(page, `Default Gauge (theme: '${theme}')`);
});
test.afterEach(async ({ page }, testInfo) => {
await scanForA11yViolations(page, testInfo.title);
});
});

View File

@@ -20,7 +20,7 @@
* at runtime from the About dialog for additional information.
*****************************************************************************/
const { test } = require('../../pluginFixtures');
const { test, scanForA11yViolations } = require('../../avpFixtures');
const percySnapshot = require('@percy/playwright');
const { expandTreePaneItemByName, createDomainObjectWithDefaults } = require('../../appActions');
const {
@@ -31,7 +31,8 @@ const { VISUAL_URL } = require('../../constants');
test.describe('Visual - Restricted Notebook', () => {
test.beforeEach(async ({ page }) => {
await startAndAddRestrictedNotebookObject(page);
const restrictedNotebook = await startAndAddRestrictedNotebookObject(page);
await page.goto(restrictedNotebook.url + '?hideTree=true&hideInspector=true');
});
test('Restricted Notebook is visually correct @addInit', async ({ page, theme }) => {
@@ -58,7 +59,7 @@ test.describe('Visual - Notebook', () => {
name: 'Dropped Overlay Plot'
});
//Open Tree
//Open Tree to perform drag
await page.getByRole('button', { name: 'Browse' }).click();
await expandTreePaneItemByName(page, myItemsFolderName);
@@ -126,4 +127,7 @@ test.describe('Visual - Notebook', () => {
// Take a snapshot
await percySnapshot(page, `Notebook Selected Entry Text Area Active (theme: '${theme}')`);
});
test.afterEach(async ({ page }, testInfo) => {
await scanForA11yViolations(page, testInfo.title);
});
});

View File

@@ -24,12 +24,12 @@
* This test is dedicated to test notification banner functionality and its accessibility attributes.
*/
const { test, expect } = require('../../pluginFixtures');
const { test, expect, scanForA11yViolations } = require('../../avpFixtures');
const percySnapshot = require('@percy/playwright');
const { createDomainObjectWithDefaults } = require('../../appActions');
const VISUAL_URL = require('../../constants').VISUAL_URL;
test.describe("Visual - Check Notification Info Banner of 'Save successful'", () => {
test.describe("Visual - Check Notification Info Banner of 'Save successful' @a11y", () => {
test.beforeEach(async ({ page }) => {
await page.goto(VISUAL_URL, { waitUntil: 'domcontentloaded' });
});
@@ -59,4 +59,7 @@ test.describe("Visual - Check Notification Info Banner of 'Save successful'", ()
expect(await page.locator('div[role="dialog"]').isVisible()).toBe(false);
await percySnapshot(page, `Notification banner dismissed (theme: '${theme}')`);
});
test.afterEach(async ({ page }, testInfo) => {
await scanForA11yViolations(page, testInfo.title);
});
});

View File

@@ -20,7 +20,7 @@
* at runtime from the About dialog for additional information.
*****************************************************************************/
const { test } = require('../../pluginFixtures');
const { test, scanForA11yViolations } = require('../../avpFixtures');
const {
setBoundsToSpanAllActivities,
setDraftStatusForPlan
@@ -32,7 +32,7 @@ const examplePlanSmall = require('../../test-data/examplePlans/ExamplePlan_Small
const snapshotScope = '.l-shell__pane-main .l-pane__contents';
test.describe('Visual - Planning', () => {
test.describe('Visual - Planning @a11y', () => {
test.beforeEach(async ({ page }) => {
await page.goto(VISUAL_URL, { waitUntil: 'domcontentloaded' });
});
@@ -97,4 +97,7 @@ test.describe('Visual - Planning', () => {
scope: snapshotScope
});
});
test.afterEach(async ({ page }, testInfo) => {
await scanForA11yViolations(page, testInfo.title);
});
});

View File

@@ -24,14 +24,14 @@
This test suite is dedicated to tests which verify search functionality.
*/
const { test, expect } = require('../../pluginFixtures');
const { test, expect, scanForA11yViolations } = require('../../avpFixtures');
const { createDomainObjectWithDefaults } = require('../../appActions');
const { VISUAL_URL } = require('../../constants');
const percySnapshot = require('@percy/playwright');
test.describe('Grand Search', () => {
let clock;
test.describe('Grand Search @a11y', () => {
let conditionWidget;
let displayLayout;
test.beforeEach(async ({ page }) => {
await page.goto(VISUAL_URL, { waitUntil: 'domcontentloaded' });
@@ -41,9 +41,9 @@ test.describe('Grand Search', () => {
name: 'Visual Test Display Layout'
});
clock = await createDomainObjectWithDefaults(page, {
type: 'Clock',
name: 'Visual Test Clock',
conditionWidget = await createDomainObjectWithDefaults(page, {
type: 'Condition Widget',
name: 'Visual Condition Widget',
parent: displayLayout.uuid
});
});
@@ -52,29 +52,27 @@ test.describe('Grand Search', () => {
page,
theme
}) => {
const searchInput = page.getByRole('searchbox', { name: 'Search Input' });
const searchResults = page.getByRole('searchbox', { name: 'OpenMCT Search' });
// Navigate to display layout
await page.goto(displayLayout.url);
// Search for the clock object
await searchInput.click();
await searchInput.fill(clock.name);
await expect(searchResults.getByText('Visual Test Clock')).toBeVisible();
// Search for the object
await page.getByRole('searchbox', { name: 'Search Input' }).click();
await page.getByRole('searchbox', { name: 'Search Input' }).fill(conditionWidget.name);
await expect(page.getByLabel('Search Result').getByText(conditionWidget.name)).toBeVisible();
//Searching for an object returns that object in the grandsearch
await percySnapshot(page, `Searching for Clock Object (theme: '${theme}')`);
await percySnapshot(page, `Searching for Object (theme: '${theme}')`);
// Enter Edit mode on the Display Layout
await page.getByRole('button', { name: 'Edit' }).click();
// Navigate to the clock object while in edit mode on the display layout
await searchInput.click();
await searchResults.getByText('Visual Test Clock').click();
// Navigate to the object while in edit mode on the display layout
await page.getByRole('searchbox', { name: 'Search Input' }).click();
await page.getByLabel('Search Result').getByText(conditionWidget.name).click();
await percySnapshot(
page,
`Preview for clock should display when editing enabled and search item clicked (theme: '${theme}')`
`Preview should display when editing enabled and search item clicked (theme: '${theme}')`
);
// Close the preview
@@ -88,17 +86,20 @@ test.describe('Grand Search', () => {
await page.getByRole('button', { name: 'Save' }).click();
await page.getByRole('listitem', { name: 'Save and Finish Editing' }).click();
// Search for the clock object
await searchInput.click();
await searchInput.fill(clock.name);
await expect(searchResults.getByText('Visual Test Clock')).toBeVisible();
// Search for the object
await page.getByRole('searchbox', { name: 'Search Input' }).click();
await page.getByRole('searchbox', { name: 'Search Input' }).fill(conditionWidget.name);
await expect(page.getByLabel('Search Result').getByText(conditionWidget.name)).toBeVisible();
// Navigate to the clock object while not in edit mode on the display layout
await searchResults.getByText('Visual Test Clock').click();
// Navigate to the object while not in edit mode on the display layout
await page.getByLabel('Search Result').getByText(conditionWidget.name).click();
await percySnapshot(
page,
`Clicking on search results should navigate to them if not editing (theme: '${theme}')`
);
});
test.afterEach(async ({ page }, testInfo) => {
await scanForA11yViolations(page, testInfo.title);
});
});