diff --git a/.circleci/config.yml b/.circleci/config.yml index 5422dd9f63..3268c660a6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,7 +23,7 @@ commands: - node/install: install-npm: true node-version: << parameters.node-version >> - - run: npm install + - run: npm install --prefer-offline --no-audit --progress=false restore_cache_cmd: description: "Custom command for restoring cache with the ability to bust cache. When BUST_CACHE is set to true, jobs will not restore cache" parameters: @@ -128,16 +128,30 @@ jobs: suite: type: string executor: pw-focal-development + parallelism: 4 steps: - build_and_install: node-version: <> - - run: npx playwright install - - run: npm run test:e2e:<> + - run: SHARD="$((${CIRCLE_NODE_INDEX}+1))"; npm run test:e2e:<> -- --shard=${SHARD}/${CIRCLE_NODE_TOTAL} - store_test_results: path: test-results/results.xml - store_artifacts: path: test-results - generate_and_store_version_and_filesystem_artifacts + perf-test: + parameters: + node-version: + type: string + executor: pw-focal-development + steps: + - build_and_install: + node-version: <> + - run: npm run test:perf + - store_test_results: + path: test-results/results.xml + - store_artifacts: + path: test-results + - generate_and_store_version_and_filesystem_artifacts workflows: overall-circleci-commit-status: #These jobs run on every commit jobs: @@ -150,10 +164,6 @@ workflows: browser: ChromeHeadless post-steps: - upload_code_covio - - unit-test: - name: node16-chrome - node-version: lts/gallium - browser: ChromeHeadless - unit-test: name: node18-chrome node-version: "18" @@ -162,6 +172,8 @@ workflows: name: e2e-ci node-version: lts/gallium suite: ci + - perf-test: + node-version: lts/gallium the-nightly: #These jobs do not run on PRs, but against master at night jobs: - unit-test: diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 747ec79bb6..2fd6b80917 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -41,6 +41,7 @@ assignees: '' - [ ] Does this impact a critical component? - [ ] Is this just a visual bug with no functional impact? - [ ] Does this block the execution of e2e tests? +- [ ] Does this have an impact on Performance? #### Additional Information diff --git a/e2e/playwright-ci.config.js b/e2e/playwright-ci.config.js index 92b55f17d8..0e8f74e8cd 100644 --- a/e2e/playwright-ci.config.js +++ b/e2e/playwright-ci.config.js @@ -9,6 +9,7 @@ const { devices } = require('@playwright/test'); const config = { retries: 1, testDir: 'tests', + testIgnore: '**/*.perf.spec.js', //Ignore performance tests and define in playwright-perfromance.config.js timeout: 60 * 1000, webServer: { command: 'npm run start', @@ -16,14 +17,15 @@ const config = { timeout: 200 * 1000, reuseExistingServer: !process.env.CI }, + maxFailures: process.env.CI ? 5 : undefined, //Limits failures to 5 to reduce CI Waste workers: 2, //Limit to 2 for CircleCI Agent use: { baseURL: 'http://localhost:8080/', headless: true, ignoreHTTPSErrors: true, - screenshot: 'on', - trace: 'on', - video: 'on' + screenshot: 'only-on-failure', + trace: 'on-first-retry', + video: 'on-first-retry' }, projects: [ { @@ -53,8 +55,11 @@ const config = { ], reporter: [ ['list'], + ['html', { + open: 'never', + outputFolder: '../test-results/html/' + }], ['junit', { outputFile: 'test-results/results.xml' }], - ['allure-playwright'], ['github'] ] }; diff --git a/e2e/playwright-local.config.js b/e2e/playwright-local.config.js index fe3145f310..124b842643 100644 --- a/e2e/playwright-local.config.js +++ b/e2e/playwright-local.config.js @@ -9,6 +9,7 @@ const { devices } = require('@playwright/test'); const config = { retries: 0, testDir: 'tests', + testIgnore: '**/*.perf.spec.js', timeout: 30 * 1000, webServer: { command: 'npm run start', @@ -22,9 +23,9 @@ const config = { baseURL: 'http://localhost:8080/', headless: false, ignoreHTTPSErrors: true, - screenshot: 'on', - trace: 'on', - video: 'on' + screenshot: 'only-on-failure', + trace: 'retain-on-failure', + video: 'retain-on-failure' }, projects: [ { @@ -54,7 +55,10 @@ const config = { ], reporter: [ ['list'], - ['allure-playwright'] + ['html', { + open: 'on-failure', + outputFolder: '../test-results' + }] ] }; diff --git a/e2e/playwright-performance.config.js b/e2e/playwright-performance.config.js new file mode 100644 index 0000000000..a718f1df03 --- /dev/null +++ b/e2e/playwright-performance.config.js @@ -0,0 +1,41 @@ +/* eslint-disable no-undef */ +// playwright.config.js +// @ts-check + +/** @type {import('@playwright/test').PlaywrightTestConfig} */ +const config = { + retries: 0, + testDir: 'tests/performance/', + timeout: 30 * 1000, + workers: 1, //Only run in serial with 1 worker + webServer: { + command: 'npm run start', + port: 8080, + timeout: 200 * 1000, + reuseExistingServer: !process.env.CI + }, + use: { + browserName: "chromium", + baseURL: 'http://localhost:8080/', + headless: Boolean(process.env.CI), //Only if running locally + ignoreHTTPSErrors: true, + screenshot: 'off', + trace: 'off', + video: 'off' + }, + projects: [ + { + name: 'chrome', + use: { + browserName: 'chromium' + } + } + ], + reporter: [ + ['list'], + ['junit', { outputFile: 'test-results/results.xml' }], + ['json', { outputFile: 'test-results/results.json' }] + ] +}; + +module.exports = config; diff --git a/e2e/test-data/PerformanceDisplayLayout.json b/e2e/test-data/PerformanceDisplayLayout.json new file mode 100644 index 0000000000..eebc7635ad --- /dev/null +++ b/e2e/test-data/PerformanceDisplayLayout.json @@ -0,0 +1 @@ +{"openmct":{"21338566-d472-4377-aed1-21b79272c8de":{"identifier":{"key":"21338566-d472-4377-aed1-21b79272c8de","namespace":""},"name":"Performance Display Layout","type":"layout","composition":[{"key":"644c2e47-2903-475f-8a4a-6be1588ee02f","namespace":""}],"configuration":{"items":[{"width":32,"height":18,"x":1,"y":1,"identifier":{"key":"644c2e47-2903-475f-8a4a-6be1588ee02f","namespace":""},"hasFrame":true,"fontSize":"default","font":"default","type":"subobject-view","id":"5aeb5a71-3149-41ed-9d8a-d34b0a18b053"}],"layoutGrid":[10,10]},"modified":1652228997384,"location":"mine","persisted":1652228997384},"644c2e47-2903-475f-8a4a-6be1588ee02f":{"identifier":{"key":"644c2e47-2903-475f-8a4a-6be1588ee02f","namespace":""},"name":"Performance Example Imagery","type":"example.imagery","configuration":{"imageLocation":"","imageLoadDelayInMilliSeconds":20000,"imageSamples":[]},"telemetry":{"values":[{"name":"Name","key":"name"},{"name":"Time","key":"utc","format":"utc","hints":{"domain":2}},{"name":"Local Time","key":"local","format":"local-format","hints":{"domain":1}},{"name":"Image","key":"url","format":"image","hints":{"image":1}},{"name":"Image Download Name","key":"imageDownloadName","format":"imageDownloadName","hints":{"imageDownloadName":1}}]},"modified":1652228997375,"location":"21338566-d472-4377-aed1-21b79272c8de","persisted":1652228997375}},"rootId":"21338566-d472-4377-aed1-21b79272c8de"} \ No newline at end of file diff --git a/e2e/test-data/PerformanceNotebook.json b/e2e/test-data/PerformanceNotebook.json new file mode 100644 index 0000000000..ae08431874 --- /dev/null +++ b/e2e/test-data/PerformanceNotebook.json @@ -0,0 +1 @@ +{"openmct":{"6d2fa9fd-f2aa-461a-a1e1-164ac44bec9d":{"identifier":{"key":"6d2fa9fd-f2aa-461a-a1e1-164ac44bec9d","namespace":""},"name":"Performance Notebook","type":"notebook","configuration":{"defaultSort":"oldest","entries":{"3e31c412-33ba-4757-8ade-e9821f6ba321":{"8c8f6035-631c-45af-8c24-786c60295335":[{"id":"entry-1652815305457","createdOn":1652815305457,"createdBy":"","text":"Existing Entry 1","embeds":[]},{"id":"entry-1652815313465","createdOn":1652815313465,"createdBy":"","text":"Existing Entry 2","embeds":[]},{"id":"entry-1652815399955","createdOn":1652815399955,"createdBy":"","text":"Existing Entry 3","embeds":[]}]}},"imageMigrationVer":"v1","pageTitle":"Page","sections":[{"id":"3e31c412-33ba-4757-8ade-e9821f6ba321","isDefault":false,"isSelected":false,"name":"Section1","pages":[{"id":"8c8f6035-631c-45af-8c24-786c60295335","isDefault":false,"isSelected":false,"name":"Page1","pageTitle":"Page"},{"id":"36555942-c9aa-439c-bbdb-0aaf50db50f5","isDefault":false,"isSelected":false,"name":"Page2","pageTitle":"Page"}],"sectionTitle":"Section"},{"id":"dab0bd1d-2c5a-405c-987f-107123d6189a","isDefault":false,"isSelected":true,"name":"Section2","pages":[{"id":"f625a86a-cb99-4898-8082-80543c8de534","isDefault":false,"isSelected":false,"name":"Page1","pageTitle":"Page"},{"id":"e77ef810-f785-42a7-942e-07e999b79c59","isDefault":false,"isSelected":true,"name":"Page2","pageTitle":"Page"}],"sectionTitle":"Section"}],"sectionTitle":"Section","type":"General","showTime":"0"},"modified":1652815915219,"location":"mine","persisted":1652815915222}},"rootId":"6d2fa9fd-f2aa-461a-a1e1-164ac44bec9d"} \ No newline at end of file diff --git a/e2e/tests/moveObjects.e2e.spec.js b/e2e/tests/moveObjects.e2e.spec.js index 81a7bc9b7a..d788192cb7 100644 --- a/e2e/tests/moveObjects.e2e.spec.js +++ b/e2e/tests/moveObjects.e2e.spec.js @@ -39,6 +39,10 @@ test.describe('Move item tests', () => { await page.locator('text=Properties Title Notes >> input[type="text"]').click(); await page.locator('text=Properties Title Notes >> input[type="text"]').fill(folder1); + + // Click on My Items in Tree. Workaround for https://github.com/nasa/openmct/issues/5184 + await page.click('form[name="mctForm"] a:has-text("My Items")'); + await Promise.all([ page.waitForNavigation(), page.locator('text=OK').click(), @@ -54,6 +58,10 @@ test.describe('Move item tests', () => { await page.locator('li.icon-folder').click(); await page.locator('text=Properties Title Notes >> input[type="text"]').click(); await page.locator('text=Properties Title Notes >> input[type="text"]').fill(folder2); + + // Click on My Items in Tree. Workaround for https://github.com/nasa/openmct/issues/5184 + await page.click('form[name="mctForm"] a:has-text("My Items")'); + await Promise.all([ page.waitForNavigation(), page.locator('text=OK').click(), @@ -72,10 +80,8 @@ test.describe('Move item tests', () => { }); await page.locator('li.icon-move').click(); await page.locator('form[name="mctForm"] >> text=My Items').click(); - await Promise.all([ - page.waitForNavigation(), - page.locator('text=OK').click() - ]); + + await page.locator('text=OK').click(); // Expect that Folder 2 is in My Items, the root folder expect(page.locator(`text=My Items >> nth=0:has(text=${folder2})`)).toBeTruthy(); @@ -90,10 +96,11 @@ test.describe('Move item tests', () => { await page.locator('li:has-text("Telemetry Table")').click(); await page.locator('text=Properties Title Notes >> input[type="text"]').click(); await page.locator('text=Properties Title Notes >> input[type="text"]').fill(telemetryTable); - await Promise.all([ - page.waitForNavigation(), - page.locator('text=OK').click() - ]); + + // Click on My Items in Tree. Workaround for https://github.com/nasa/openmct/issues/5184 + await page.click('form[name="mctForm"] a:has-text("My Items")'); + + await page.locator('text=OK').click(); // Finish editing and save Telemetry Table await page.locator('.c-button--menu.c-button--major.icon-save').click(); @@ -114,10 +121,7 @@ test.describe('Move item tests', () => { // Continue test regardless of assertion and create it in My Items await page.locator('form[name="mctForm"] >> text=My Items').click(); - await Promise.all([ - page.waitForNavigation(), - page.locator('text=OK').click() - ]); + await page.locator('text=OK').click(); // Open My Items await page.locator('text=Open MCT My Items >> span').nth(3).click(); diff --git a/e2e/tests/performance/imagery.perf.spec.js b/e2e/tests/performance/imagery.perf.spec.js new file mode 100644 index 0000000000..0e75e52af1 --- /dev/null +++ b/e2e/tests/performance/imagery.perf.spec.js @@ -0,0 +1,177 @@ +/***************************************************************************** + * Open MCT, Copyright (c) 2014-2022, United States Government + * as represented by the Administrator of the National Aeronautics and Space + * Administration. All rights reserved. + * + * Open MCT is licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * Open MCT includes source code licensed under additional open source + * licenses. See the Open Source Licenses file (LICENSES.md) included with + * this source code distribution or the Licensing information page available + * at runtime from the About dialog for additional information. + *****************************************************************************/ + +/* +This test suite is dedicated to performance tests to ensure that testability of performance +is not broken upstream on Open MCT. Any assumptions made downstream will be tested here + +TODO: + - Update resolution of performance config + - Add Performance Observer on init to push all performance marks + - Move client CDP connection to before or to a fixture + - + +*/ + +const { test, expect } = require('@playwright/test'); + +const filePath = 'e2e/test-data/PerformanceDisplayLayout.json'; + +test.describe('Performance tests', () => { + test.beforeEach(async ({ page, browser }, testInfo) => { + // Go to baseURL + await page.goto('/', { waitUntil: 'networkidle' }); + + // Click a:has-text("My Items") + await page.locator('a:has-text("My Items")').click({ + button: 'right' + }); + + // Click text=Import from JSON + await page.locator('text=Import from JSON').click(); + + // Upload Performance Display Layout.json + await page.setInputFiles('#fileElem', filePath); + + // Click text=OK + await page.locator('text=OK').click(); + + await expect(page.locator('a:has-text("Performance Display Layout Display Layout")')).toBeVisible(); + + //Create a Chrome Performance Timeline trace to store as a test artifact + console.log("\n==== Devtools: startTracing ====\n"); + await browser.startTracing(page, { + path: `${testInfo.outputPath()}-trace.json`, + screenshots: true + }); + }); + test.afterEach(async ({ page, browser}) => { + console.log("\n==== Devtools: stopTracing ====\n"); + await browser.stopTracing(); + + /* Measurement Section + / The following section includes a block of performance measurements. + */ + //Get time difference between viewlarge actionability and evaluate time + await page.evaluate(() => (window.performance.measure("machine-time-difference", "viewlarge.start", "viewLarge.start.test"))); + + //Get StartTime + const startTime = await page.evaluate(() => window.performance.timing.navigationStart); + console.log('window.performance.timing.navigationStart', startTime); + + //Get All Performance Marks + const getAllMarksJson = await page.evaluate(() => JSON.stringify(window.performance.getEntriesByType("mark"))); + const getAllMarks = JSON.parse(getAllMarksJson); + console.log('window.performance.getEntriesByType("mark")', getAllMarks); + + //Get All Performance Measures + const getAllMeasuresJson = await page.evaluate(() => JSON.stringify(window.performance.getEntriesByType("measure"))); + const getAllMeasures = JSON.parse(getAllMeasuresJson); + console.log('window.performance.getEntriesByType("measure")', getAllMeasures); + + }); + /* The following test will navigate to a previously created Performance Display Layout and measure the + / following metrics: + / - ElementResourceTiming + / - Interaction Timing + */ + test('Embedded View Large for Imagery is performant in Fixed Time', async ({ page, browser }) => { + const client = await page.context().newCDPSession(page); + // Tell the DevTools session to record performance metrics + // https://chromedevtools.github.io/devtools-protocol/tot/Performance/#method-getMetrics + await client.send('Performance.enable'); + // Go to baseURL + await page.goto('/'); + + // Search Available after Launch + await page.locator('input[type="search"]').click(); + await page.evaluate(() => window.performance.mark("search-available")); + // Fill Search input + await page.locator('input[type="search"]').fill('Performance Display Layout'); + await page.evaluate(() => window.performance.mark("search-entered")); + //Search Result Appears and is clicked + await Promise.all([ + page.waitForNavigation(), + page.locator('a:has-text("Performance Display Layout")').first().click(), + page.evaluate(() => window.performance.mark("click-search-result")) + ]); + + //Time to Example Imagery Frame loads within Display Layout + await page.waitForSelector('.c-imagery__main-image__bg', { state: 'visible'}); + //Time to Example Imagery object loads + await page.waitForSelector('.c-imagery__main-image__background-image', { state: 'visible'}); + + //Get background-image url from background-image css prop + const backgroundImage = await page.locator('.c-imagery__main-image__background-image'); + let backgroundImageUrl = await backgroundImage.evaluate((el) => { + return window.getComputedStyle(el).getPropertyValue('background-image').match(/url\(([^)]+)\)/)[1]; + }); + backgroundImageUrl = backgroundImageUrl.slice(1, -1); //forgive me, padre + console.log('backgroundImageurl ' + backgroundImageUrl); + + //Get ResourceTiming of background-image jpg + const resourceTimingJson = await page.evaluate((bgImageUrl) => + JSON.stringify(window.performance.getEntriesByName(bgImageUrl).pop()), + backgroundImageUrl + ); + console.log('resourceTimingJson ' + resourceTimingJson); + + //Open Large view + await page.locator('button:has-text("Large View")').click(); //This action includes the performance.mark named 'viewLarge.start' + await page.evaluate(() => window.performance.mark("viewLarge.start.test")); //This is a mark only to compare evaluate timing + + //Time to Imagery Rendered in Large Frame + await page.waitForSelector('.c-imagery__main-image__bg', { state: 'visible'}); + await page.evaluate(() => window.performance.mark("background-image-frame")); + + //Time to Example Imagery object loads + await page.waitForSelector('.c-imagery__main-image__background-image', { state: 'visible'}); + await page.evaluate(() => window.performance.mark("background-image-visible")); + + // Get Current number of images in thumbstrip + await page.waitForSelector('.c-imagery__thumb'); + const thumbCount = await page.locator('.c-imagery__thumb').count(); + console.log('number of thumbs rendered ' + thumbCount); + await page.locator('.c-imagery__thumb').last().click(); + + //Get ResourceTiming of all jpg resources + const resourceTimingJson2 = await page.evaluate(() => + JSON.stringify(window.performance.getEntriesByType('resource')) + ); + const resourceTiming = JSON.parse(resourceTimingJson2); + const jpgResourceTiming = resourceTiming.find((element) => + element.name.includes('.jpg') + ); + console.log('jpgResourceTiming ' + JSON.stringify(jpgResourceTiming)); + + // Click Close Icon + await page.locator('.c-click-icon').click(); + await page.evaluate(() => window.performance.mark("view-large-close-button")); + + //await client.send('HeapProfiler.enable'); + await client.send('HeapProfiler.collectGarbage'); + + let performanceMetrics = await client.send('Performance.getMetrics'); + console.log(performanceMetrics.metrics); + + }); +}); diff --git a/e2e/tests/performance/memleak-imagery.perf.spec.js b/e2e/tests/performance/memleak-imagery.perf.spec.js new file mode 100644 index 0000000000..86f6d68a73 --- /dev/null +++ b/e2e/tests/performance/memleak-imagery.perf.spec.js @@ -0,0 +1,118 @@ +/***************************************************************************** + * Open MCT, Copyright (c) 2014-2022, United States Government + * as represented by the Administrator of the National Aeronautics and Space + * Administration. All rights reserved. + * + * Open MCT is licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * Open MCT includes source code licensed under additional open source + * licenses. See the Open Source Licenses file (LICENSES.md) included with + * this source code distribution or the Licensing information page available + * at runtime from the About dialog for additional information. + *****************************************************************************/ + +/* +This test suite is an initial example for memory leak testing using performance. This configuration and execution must +be kept separate from the traditional performance measurements to avoid any "observer" effects associated with tracing +or profiling playwright and/or the browser. + +Based on a pattern identified in https://github.com/trentmwillis/devtools-protocol-demos/blob/master/testing-demos/memory-leak-by-heap.js +and https://github.com/paulirish/automated-chrome-profiling/issues/3 + +Best path forward: https://github.com/cowchimp/headless-devtools/blob/master/src/Memory/example.js + +*/ + +const { test, expect } = require('@playwright/test'); + +const filePath = 'e2e/test-data/PerformanceDisplayLayout.json'; + +test.describe.skip('Memory Performance tests', () => { + test.beforeEach(async ({ page, browser }, testInfo) => { + // Go to baseURL + await page.goto('/', { waitUntil: 'networkidle' }); + + // Click a:has-text("My Items") + await page.locator('a:has-text("My Items")').click({ + button: 'right' + }); + + // Click text=Import from JSON + await page.locator('text=Import from JSON').click(); + + // Upload Performance Display Layout.json + await page.setInputFiles('#fileElem', filePath); + + // Click text=OK + await page.locator('text=OK').click(); + + await expect(page.locator('a:has-text("Performance Display Layout Display Layout")')).toBeVisible(); + }); + + test.skip('Embedded View Large for Imagery is performant in Fixed Time', async ({ page, browser }) => { + + await page.goto('/', {waitUntil: 'networkidle'}); + + // To to Search Available after Launch + await page.locator('input[type="search"]').click(); + // Fill Search input + await page.locator('input[type="search"]').fill('Performance Display Layout'); + //Search Result Appears and is clicked + await Promise.all([ + page.waitForNavigation(), + page.locator('a:has-text("Performance Display Layout")').first().click() + ]); + + //Time to Example Imagery Frame loads within Display Layout + await page.waitForSelector('.c-imagery__main-image__bg', { state: 'visible'}); + //Time to Example Imagery object loads + await page.waitForSelector('.c-imagery__main-image__background-image', { state: 'visible'}); + + const client = await page.context().newCDPSession(page); + await client.send('HeapProfiler.enable'); + await client.send('HeapProfiler.startSampling'); + // await client.send('HeapProfiler.collectGarbage'); + await client.send('Performance.enable'); + + let performanceMetricsBefore = await client.send('Performance.getMetrics'); + console.log(performanceMetricsBefore.metrics); + + //await client.send('Performance.disable'); + + //Open Large view + await page.locator('button:has-text("Large View")').click(); + await client.send('HeapProfiler.takeHeapSnapshot'); + + //Time to Imagery Rendered in Large Frame + await page.waitForSelector('.c-imagery__main-image__bg', { state: 'visible'}); + + //Time to Example Imagery object loads + await page.waitForSelector('.c-imagery__main-image__background-image', { state: 'visible'}); + + // Click Close Icon + await page.locator('.c-click-icon').click(); + + //Time to Example Imagery Frame loads within Display Layout + await page.waitForSelector('.c-imagery__main-image__bg', { state: 'visible'}); + //Time to Example Imagery object loads + await page.waitForSelector('.c-imagery__main-image__background-image', { state: 'visible'}); + + await client.send('HeapProfiler.collectGarbage'); + //await client.send('Performance.enable'); + + let performanceMetricsAfter = await client.send('Performance.getMetrics'); + console.log(performanceMetricsAfter.metrics); + + //await client.send('Performance.disable'); + + }); +}); diff --git a/e2e/tests/performance/notebook.perf.spec.js b/e2e/tests/performance/notebook.perf.spec.js new file mode 100644 index 0000000000..14c74bd32d --- /dev/null +++ b/e2e/tests/performance/notebook.perf.spec.js @@ -0,0 +1,158 @@ +/***************************************************************************** + * Open MCT, Copyright (c) 2014-2022, United States Government + * as represented by the Administrator of the National Aeronautics and Space + * Administration. All rights reserved. + * + * Open MCT is licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * Open MCT includes source code licensed under additional open source + * licenses. See the Open Source Licenses file (LICENSES.md) included with + * this source code distribution or the Licensing information page available + * at runtime from the About dialog for additional information. + *****************************************************************************/ + +/* +This test suite is dedicated to performance tests to ensure that testability of performance +is not broken upstream on Open MCT. Any assumptions made downstream will be tested here. + +TODO: + - Update resolution of performance config + - Add Performance Observer on init to push all performance marks + - Move client CDP connection to before or to a fixture + +*/ + +const { test, expect } = require('@playwright/test'); + +const notebookFilePath = 'e2e/test-data/PerformanceNotebook.json'; + +test.describe('Performance tests', () => { + test.beforeEach(async ({ page, browser }, testInfo) => { + // Go to baseURL + await page.goto('/', { waitUntil: 'networkidle' }); + + // Click a:has-text("My Items") + await page.locator('a:has-text("My Items")').click({ + button: 'right' + }); + + // Click text=Import from JSON + await page.locator('text=Import from JSON').click(); + + // Upload Performance Display Layout.json + await page.setInputFiles('#fileElem', notebookFilePath); + + // TODO Fix this + await page.locator('text=OK >> nth=1').click(); + + await expect(page.locator('a:has-text("Performance Notebook")')).toBeVisible(); + + //Create a Chrome Performance Timeline trace to store as a test artifact + console.log("\n==== Devtools: startTracing ====\n"); + await browser.startTracing(page, { + path: `${testInfo.outputPath()}-trace.json`, + screenshots: true + }); + }); + test.afterEach(async ({ page, browser}) => { + console.log("\n==== Devtools: stopTracing ====\n"); + await browser.stopTracing(); + + /* Measurement Section + / The following section includes a block of performance measurements. + */ + const startTime = await page.evaluate(() => window.performance.timing.navigationStart); + console.log('window.performance.timing.navigationStart', startTime); + + //Get All Performance Marks + const getAllMarksJson = await page.evaluate(() => JSON.stringify(window.performance.getEntriesByType("mark"))); + const getAllMarks = JSON.parse(getAllMarksJson); + console.log('window.performance.getEntriesByType("mark")', getAllMarks); + + //Get All Performance Measures + const getAllMeasuresJson = await page.evaluate(() => JSON.stringify(window.performance.getEntriesByType("measure"))); + const getAllMeasures = JSON.parse(getAllMeasuresJson); + console.log('window.performance.getEntriesByType("measure")', getAllMeasures); + + }); + /* The following test will navigate to a previously created Performance Display Layout and measure the + / following metrics: + / - ElementResourceTiming + / - Interaction Timing + */ + test('Notebook Search, Add Entry, Update Entry are performant', async ({ page, browser }) => { + const client = await page.context().newCDPSession(page); + // Tell the DevTools session to record performance metrics + // https://chromedevtools.github.io/devtools-protocol/tot/Performance/#method-getMetrics + await client.send('Performance.enable'); + // Go to baseURL + await page.goto('/'); + + // To to Search Available after Launch + await page.locator('input[type="search"]').click(); + await page.evaluate(() => window.performance.mark("search-available")); + // Fill Search input + await page.locator('input[type="search"]').fill('Performance Notebook'); + await page.evaluate(() => window.performance.mark("search-entered")); + //Search Result Appears and is clicked + await Promise.all([ + page.waitForNavigation(), + page.locator('a:has-text("Performance Notebook")').first().click(), + page.evaluate(() => window.performance.mark("click-search-result")) + ]); + + await page.waitForSelector('.c-tree__item c-tree-and-search__loading loading', {state: 'hidden'}); + await page.evaluate(() => window.performance.mark("search-spinner-gone")); + + await page.waitForSelector('.l-browse-bar__object-name', { state: 'visible'}); + await page.evaluate(() => window.performance.mark("object-title-appears")); + + await page.waitForSelector('.c-notebook__entry >> nth=0', { state: 'visible'}); + await page.evaluate(() => window.performance.mark("notebook-entry-appears")); + + // Click Add new Notebook Entry + await page.locator('.c-notebook__drag-area').click(); + await page.evaluate(() => window.performance.mark("new-notebook-entry-created")); + + // Enter Notebook Entry text + await page.locator('div.c-ne__text').last().fill('New Entry'); + await page.keyboard.press('Enter'); + await page.evaluate(() => window.performance.mark("new-notebook-entry-filled")); + + //Individual Notebook Entry Search + await page.evaluate(() => window.performance.mark("notebook-search-start")); + await page.locator('.c-notebook__search >> input').fill('Existing Entry'); + await page.evaluate(() => window.performance.mark("notebook-search-filled")); + await page.waitForSelector('text=Search Results (3)', { state: 'visible'}); + await page.evaluate(() => window.performance.mark("notebook-search-processed")); + await page.waitForSelector('.c-notebook__entry >> nth=2', { state: 'visible'}); + await page.evaluate(() => window.performance.mark("notebook-search-processed")); + + //Clear Search + await page.locator('.c-search.c-notebook__search .c-search__clear-input').click(); + await page.evaluate(() => window.performance.mark("notebook-search-processed")); + + // Hover on Last + await page.evaluate(() => window.performance.mark("new-notebook-entry-delete")); + await page.locator('div.c-ne__time-and-content').last().hover(); + await page.locator('button[title="Delete this entry"]').last().click(); + await page.locator('button:has-text("Ok")').click(); + await page.waitForSelector('.c-notebook__entry >> nth=3', { state: 'detached'}); + await page.evaluate(() => window.performance.mark("new-notebook-entry-deleted")); + + //await client.send('HeapProfiler.enable'); + await client.send('HeapProfiler.collectGarbage'); + + let performanceMetrics = await client.send('Performance.getMetrics'); + console.log(performanceMetrics.metrics); + }); +}); diff --git a/e2e/tests/plugins/condition/condition.e2e.spec.js b/e2e/tests/plugins/condition/condition.e2e.spec.js index 5b4cc14850..68f31eed01 100644 --- a/e2e/tests/plugins/condition/condition.e2e.spec.js +++ b/e2e/tests/plugins/condition/condition.e2e.spec.js @@ -42,6 +42,9 @@ test('Create new Condition Set object and store @localStorage', async ({ page, c // Click text=Condition Set await page.click('text=Condition Set'); + // Click on My Items in Tree. Workaround for https://github.com/nasa/openmct/issues/5184 + await page.click('form[name="mctForm"] a:has-text("My Items")'); + // Click text=OK await Promise.all([ page.waitForNavigation(), diff --git a/e2e/tests/plugins/plot/autoscale.e2e.spec.js-snapshots/autoscale-canvas-panned-chrome.png b/e2e/tests/plugins/plot/autoscale.e2e.spec.js-snapshots/autoscale-canvas-panned-chrome.png index 3170e08516..4b546e1878 100644 Binary files a/e2e/tests/plugins/plot/autoscale.e2e.spec.js-snapshots/autoscale-canvas-panned-chrome.png and b/e2e/tests/plugins/plot/autoscale.e2e.spec.js-snapshots/autoscale-canvas-panned-chrome.png differ diff --git a/e2e/tests/plugins/plot/autoscale.e2e.spec.js-snapshots/autoscale-canvas-prepan-chrome.png b/e2e/tests/plugins/plot/autoscale.e2e.spec.js-snapshots/autoscale-canvas-prepan-chrome.png index f9b3595d0c..c0f293bd16 100644 Binary files a/e2e/tests/plugins/plot/autoscale.e2e.spec.js-snapshots/autoscale-canvas-prepan-chrome.png and b/e2e/tests/plugins/plot/autoscale.e2e.spec.js-snapshots/autoscale-canvas-prepan-chrome.png differ diff --git a/package.json b/package.json index 5c58eb927a..aaada4d8f4 100644 --- a/package.json +++ b/package.json @@ -6,14 +6,13 @@ "@babel/eslint-parser": "7.16.3", "@braintree/sanitize-url": "6.0.0", "@percy/cli": "1.2.1", - "@percy/playwright": "1.0.3", + "@percy/playwright": "1.0.4", "@playwright/test": "1.21.1", "@types/eventemitter3": "^1.0.0", "@types/jasmine": "^4.0.1", "@types/karma": "^6.3.2", "@types/lodash": "^4.14.178", "@types/mocha": "^9.1.0", - "allure-playwright": "2.0.0-beta.15", "babel-loader": "8.2.3", "babel-plugin-istanbul": "6.1.1", "comma-separated-values": "3.6.4", @@ -92,11 +91,12 @@ "test": "cross-env NODE_OPTIONS=\"--max_old_space_size=4096\" karma start --single-run", "test:firefox": "cross-env NODE_OPTIONS=\"--max_old_space_size=4096\" karma start --single-run --browsers=FirefoxHeadless", "test:debug": "cross-env NODE_ENV=debug karma start --no-single-run", - "test:e2e:ci": "npx playwright test --config=e2e/playwright-ci.config.js --project=chrome smoke default condition timeConductor branding clock exampleImagery", + "test:e2e:ci": "npx playwright test --config=e2e/playwright-ci.config.js --project=chrome smoke branding default condition timeConductor clock exampleImagery notebook persistence performance", "test:e2e:local": "npx playwright test --config=e2e/playwright-local.config.js --project=chrome", "test:e2e:updatesnapshots": "npx playwright test --config=e2e/playwright-local.config.js --project=chrome --grep @snapshot --update-snapshots", "test:e2e:visual": "percy exec --config ./e2e/.percy.yml -- npx playwright test --config=e2e/playwright-visual.config.js", "test:e2e:full": "npx playwright test --config=e2e/playwright-ci.config.js", + "test:perf": "npx playwright test --config=e2e/playwright-performance.config.js", "test:watch": "cross-env NODE_OPTIONS=\"--max_old_space_size=4096\" karma start --no-single-run", "jsdoc": "jsdoc -c jsdoc.json -R API.md -r -d dist/docs/api", "update-about-dialog-copyright": "perl -pi -e 's/20\\d\\d\\-202\\d/2014\\-2022/gm' ./src/ui/layout/AboutDialog.vue",