mirror of
https://github.com/nasa/openmct.git
synced 2024-12-18 20:57:53 +00:00
Performance tests, notebook tests, and sharding (#5236)
* Renamed test files * temp push of performance branch * comma * stash * final * rename to imagery * remove old * stash * fix name * Add plain notebook e2e and perf test * Shard and add perf to ci * Import fixtures * 3 * also off by one? * forgive me, father * update perf test name * SHARD * one mo shot * add suites * failfast * remove allure * add more testsuites * full * skip * ignore glob * headless? * skip audit to save ci time * temp push * remove allure * remove doubled test and update snapshots * update snapthos 2 * back to ci * update comments * remove notebook * updates * remove npx playwright install * Prevent deleting browsers * circleci comment * re-enable audit * Add html reporter * speed up execution * speed up CI * Add performance line to bug report * PR Comments * change dir of report * remove test * Update Tracepath. Update playwright-percy * Remove shim * Improve stability and move html report * fix space * Add more slowdown for moveObjects * Get rid of navigation event * fix missing events * review comments * last change! Co-authored-by: unlikelyzero <jchill2@gmail.com>
This commit is contained in:
John Hill
GitHub
parent
f74a35f45a
commit
c56d458ecb
@ -23,7 +23,7 @@ commands:
|
||||
- node/install:
|
||||
install-npm: true
|
||||
node-version: << parameters.node-version >>
|
||||
- run: npm install
|
||||
- run: npm install --prefer-offline --no-audit --progress=false
|
||||
restore_cache_cmd:
|
||||
description: "Custom command for restoring cache with the ability to bust cache. When BUST_CACHE is set to true, jobs will not restore cache"
|
||||
parameters:
|
||||
@ -128,16 +128,30 @@ jobs:
|
||||
suite:
|
||||
type: string
|
||||
executor: pw-focal-development
|
||||
parallelism: 4
|
||||
steps:
|
||||
- build_and_install:
|
||||
node-version: <<parameters.node-version>>
|
||||
- run: npx playwright install
|
||||
- run: npm run test:e2e:<<parameters.suite>>
|
||||
- run: SHARD="$((${CIRCLE_NODE_INDEX}+1))"; npm run test:e2e:<<parameters.suite>> -- --shard=${SHARD}/${CIRCLE_NODE_TOTAL}
|
||||
- store_test_results:
|
||||
path: test-results/results.xml
|
||||
- store_artifacts:
|
||||
path: test-results
|
||||
- generate_and_store_version_and_filesystem_artifacts
|
||||
perf-test:
|
||||
parameters:
|
||||
node-version:
|
||||
type: string
|
||||
executor: pw-focal-development
|
||||
steps:
|
||||
- build_and_install:
|
||||
node-version: <<parameters.node-version>>
|
||||
- run: npm run test:perf
|
||||
- store_test_results:
|
||||
path: test-results/results.xml
|
||||
- store_artifacts:
|
||||
path: test-results
|
||||
- generate_and_store_version_and_filesystem_artifacts
|
||||
workflows:
|
||||
overall-circleci-commit-status: #These jobs run on every commit
|
||||
jobs:
|
||||
@ -150,10 +164,6 @@ workflows:
|
||||
browser: ChromeHeadless
|
||||
post-steps:
|
||||
- upload_code_covio
|
||||
- unit-test:
|
||||
name: node16-chrome
|
||||
node-version: lts/gallium
|
||||
browser: ChromeHeadless
|
||||
- unit-test:
|
||||
name: node18-chrome
|
||||
node-version: "18"
|
||||
@ -162,6 +172,8 @@ workflows:
|
||||
name: e2e-ci
|
||||
node-version: lts/gallium
|
||||
suite: ci
|
||||
- perf-test:
|
||||
node-version: lts/gallium
|
||||
the-nightly: #These jobs do not run on PRs, but against master at night
|
||||
jobs:
|
||||
- unit-test:
|
||||
|
||||
1
.github/ISSUE_TEMPLATE/bug_report.md
vendored
1
.github/ISSUE_TEMPLATE/bug_report.md
vendored
@ -41,6 +41,7 @@ assignees: ''
|
||||
- [ ] Does this impact a critical component?
|
||||
- [ ] Is this just a visual bug with no functional impact?
|
||||
- [ ] Does this block the execution of e2e tests?
|
||||
- [ ] Does this have an impact on Performance?
|
||||
|
||||
#### Additional Information
|
||||
<!--- Include any screenshots, gifs, or logs which will expedite triage -->
|
||||
|
||||
@ -9,6 +9,7 @@ const { devices } = require('@playwright/test');
|
||||
const config = {
|
||||
retries: 1,
|
||||
testDir: 'tests',
|
||||
testIgnore: '**/*.perf.spec.js', //Ignore performance tests and define in playwright-perfromance.config.js
|
||||
timeout: 60 * 1000,
|
||||
webServer: {
|
||||
command: 'npm run start',
|
||||
@ -16,14 +17,15 @@ const config = {
|
||||
timeout: 200 * 1000,
|
||||
reuseExistingServer: !process.env.CI
|
||||
},
|
||||
maxFailures: process.env.CI ? 5 : undefined, //Limits failures to 5 to reduce CI Waste
|
||||
workers: 2, //Limit to 2 for CircleCI Agent
|
||||
use: {
|
||||
baseURL: 'http://localhost:8080/',
|
||||
headless: true,
|
||||
ignoreHTTPSErrors: true,
|
||||
screenshot: 'on',
|
||||
trace: 'on',
|
||||
video: 'on'
|
||||
screenshot: 'only-on-failure',
|
||||
trace: 'on-first-retry',
|
||||
video: 'on-first-retry'
|
||||
},
|
||||
projects: [
|
||||
{
|
||||
@ -53,8 +55,11 @@ const config = {
|
||||
],
|
||||
reporter: [
|
||||
['list'],
|
||||
['html', {
|
||||
open: 'never',
|
||||
outputFolder: '../test-results/html/'
|
||||
}],
|
||||
['junit', { outputFile: 'test-results/results.xml' }],
|
||||
['allure-playwright'],
|
||||
['github']
|
||||
]
|
||||
};
|
||||
|
||||
@ -9,6 +9,7 @@ const { devices } = require('@playwright/test');
|
||||
const config = {
|
||||
retries: 0,
|
||||
testDir: 'tests',
|
||||
testIgnore: '**/*.perf.spec.js',
|
||||
timeout: 30 * 1000,
|
||||
webServer: {
|
||||
command: 'npm run start',
|
||||
@ -22,9 +23,9 @@ const config = {
|
||||
baseURL: 'http://localhost:8080/',
|
||||
headless: false,
|
||||
ignoreHTTPSErrors: true,
|
||||
screenshot: 'on',
|
||||
trace: 'on',
|
||||
video: 'on'
|
||||
screenshot: 'only-on-failure',
|
||||
trace: 'retain-on-failure',
|
||||
video: 'retain-on-failure'
|
||||
},
|
||||
projects: [
|
||||
{
|
||||
@ -54,7 +55,10 @@ const config = {
|
||||
],
|
||||
reporter: [
|
||||
['list'],
|
||||
['allure-playwright']
|
||||
['html', {
|
||||
open: 'on-failure',
|
||||
outputFolder: '../test-results'
|
||||
}]
|
||||
]
|
||||
};
|
||||
|
||||
|
||||
41
e2e/playwright-performance.config.js
Normal file
41
e2e/playwright-performance.config.js
Normal file
@ -0,0 +1,41 @@
|
||||
/* eslint-disable no-undef */
|
||||
// playwright.config.js
|
||||
// @ts-check
|
||||
|
||||
/** @type {import('@playwright/test').PlaywrightTestConfig} */
|
||||
const config = {
|
||||
retries: 0,
|
||||
testDir: 'tests/performance/',
|
||||
timeout: 30 * 1000,
|
||||
workers: 1, //Only run in serial with 1 worker
|
||||
webServer: {
|
||||
command: 'npm run start',
|
||||
port: 8080,
|
||||
timeout: 200 * 1000,
|
||||
reuseExistingServer: !process.env.CI
|
||||
},
|
||||
use: {
|
||||
browserName: "chromium",
|
||||
baseURL: 'http://localhost:8080/',
|
||||
headless: Boolean(process.env.CI), //Only if running locally
|
||||
ignoreHTTPSErrors: true,
|
||||
screenshot: 'off',
|
||||
trace: 'off',
|
||||
video: 'off'
|
||||
},
|
||||
projects: [
|
||||
{
|
||||
name: 'chrome',
|
||||
use: {
|
||||
browserName: 'chromium'
|
||||
}
|
||||
}
|
||||
],
|
||||
reporter: [
|
||||
['list'],
|
||||
['junit', { outputFile: 'test-results/results.xml' }],
|
||||
['json', { outputFile: 'test-results/results.json' }]
|
||||
]
|
||||
};
|
||||
|
||||
module.exports = config;
|
||||
1
e2e/test-data/PerformanceDisplayLayout.json
Normal file
1
e2e/test-data/PerformanceDisplayLayout.json
Normal file
@ -0,0 +1 @@
|
||||
{"openmct":{"21338566-d472-4377-aed1-21b79272c8de":{"identifier":{"key":"21338566-d472-4377-aed1-21b79272c8de","namespace":""},"name":"Performance Display Layout","type":"layout","composition":[{"key":"644c2e47-2903-475f-8a4a-6be1588ee02f","namespace":""}],"configuration":{"items":[{"width":32,"height":18,"x":1,"y":1,"identifier":{"key":"644c2e47-2903-475f-8a4a-6be1588ee02f","namespace":""},"hasFrame":true,"fontSize":"default","font":"default","type":"subobject-view","id":"5aeb5a71-3149-41ed-9d8a-d34b0a18b053"}],"layoutGrid":[10,10]},"modified":1652228997384,"location":"mine","persisted":1652228997384},"644c2e47-2903-475f-8a4a-6be1588ee02f":{"identifier":{"key":"644c2e47-2903-475f-8a4a-6be1588ee02f","namespace":""},"name":"Performance Example Imagery","type":"example.imagery","configuration":{"imageLocation":"","imageLoadDelayInMilliSeconds":20000,"imageSamples":[]},"telemetry":{"values":[{"name":"Name","key":"name"},{"name":"Time","key":"utc","format":"utc","hints":{"domain":2}},{"name":"Local Time","key":"local","format":"local-format","hints":{"domain":1}},{"name":"Image","key":"url","format":"image","hints":{"image":1}},{"name":"Image Download Name","key":"imageDownloadName","format":"imageDownloadName","hints":{"imageDownloadName":1}}]},"modified":1652228997375,"location":"21338566-d472-4377-aed1-21b79272c8de","persisted":1652228997375}},"rootId":"21338566-d472-4377-aed1-21b79272c8de"}
|
||||
1
e2e/test-data/PerformanceNotebook.json
Normal file
1
e2e/test-data/PerformanceNotebook.json
Normal file
@ -0,0 +1 @@
|
||||
{"openmct":{"6d2fa9fd-f2aa-461a-a1e1-164ac44bec9d":{"identifier":{"key":"6d2fa9fd-f2aa-461a-a1e1-164ac44bec9d","namespace":""},"name":"Performance Notebook","type":"notebook","configuration":{"defaultSort":"oldest","entries":{"3e31c412-33ba-4757-8ade-e9821f6ba321":{"8c8f6035-631c-45af-8c24-786c60295335":[{"id":"entry-1652815305457","createdOn":1652815305457,"createdBy":"","text":"Existing Entry 1","embeds":[]},{"id":"entry-1652815313465","createdOn":1652815313465,"createdBy":"","text":"Existing Entry 2","embeds":[]},{"id":"entry-1652815399955","createdOn":1652815399955,"createdBy":"","text":"Existing Entry 3","embeds":[]}]}},"imageMigrationVer":"v1","pageTitle":"Page","sections":[{"id":"3e31c412-33ba-4757-8ade-e9821f6ba321","isDefault":false,"isSelected":false,"name":"Section1","pages":[{"id":"8c8f6035-631c-45af-8c24-786c60295335","isDefault":false,"isSelected":false,"name":"Page1","pageTitle":"Page"},{"id":"36555942-c9aa-439c-bbdb-0aaf50db50f5","isDefault":false,"isSelected":false,"name":"Page2","pageTitle":"Page"}],"sectionTitle":"Section"},{"id":"dab0bd1d-2c5a-405c-987f-107123d6189a","isDefault":false,"isSelected":true,"name":"Section2","pages":[{"id":"f625a86a-cb99-4898-8082-80543c8de534","isDefault":false,"isSelected":false,"name":"Page1","pageTitle":"Page"},{"id":"e77ef810-f785-42a7-942e-07e999b79c59","isDefault":false,"isSelected":true,"name":"Page2","pageTitle":"Page"}],"sectionTitle":"Section"}],"sectionTitle":"Section","type":"General","showTime":"0"},"modified":1652815915219,"location":"mine","persisted":1652815915222}},"rootId":"6d2fa9fd-f2aa-461a-a1e1-164ac44bec9d"}
|
||||
@ -39,6 +39,10 @@ test.describe('Move item tests', () => {
|
||||
|
||||
await page.locator('text=Properties Title Notes >> input[type="text"]').click();
|
||||
await page.locator('text=Properties Title Notes >> input[type="text"]').fill(folder1);
|
||||
|
||||
// Click on My Items in Tree. Workaround for https://github.com/nasa/openmct/issues/5184
|
||||
await page.click('form[name="mctForm"] a:has-text("My Items")');
|
||||
|
||||
await Promise.all([
|
||||
page.waitForNavigation(),
|
||||
page.locator('text=OK').click(),
|
||||
@ -54,6 +58,10 @@ test.describe('Move item tests', () => {
|
||||
await page.locator('li.icon-folder').click();
|
||||
await page.locator('text=Properties Title Notes >> input[type="text"]').click();
|
||||
await page.locator('text=Properties Title Notes >> input[type="text"]').fill(folder2);
|
||||
|
||||
// Click on My Items in Tree. Workaround for https://github.com/nasa/openmct/issues/5184
|
||||
await page.click('form[name="mctForm"] a:has-text("My Items")');
|
||||
|
||||
await Promise.all([
|
||||
page.waitForNavigation(),
|
||||
page.locator('text=OK').click(),
|
||||
@ -72,10 +80,8 @@ test.describe('Move item tests', () => {
|
||||
});
|
||||
await page.locator('li.icon-move').click();
|
||||
await page.locator('form[name="mctForm"] >> text=My Items').click();
|
||||
await Promise.all([
|
||||
page.waitForNavigation(),
|
||||
page.locator('text=OK').click()
|
||||
]);
|
||||
|
||||
await page.locator('text=OK').click();
|
||||
|
||||
// Expect that Folder 2 is in My Items, the root folder
|
||||
expect(page.locator(`text=My Items >> nth=0:has(text=${folder2})`)).toBeTruthy();
|
||||
@ -90,10 +96,11 @@ test.describe('Move item tests', () => {
|
||||
await page.locator('li:has-text("Telemetry Table")').click();
|
||||
await page.locator('text=Properties Title Notes >> input[type="text"]').click();
|
||||
await page.locator('text=Properties Title Notes >> input[type="text"]').fill(telemetryTable);
|
||||
await Promise.all([
|
||||
page.waitForNavigation(),
|
||||
page.locator('text=OK').click()
|
||||
]);
|
||||
|
||||
// Click on My Items in Tree. Workaround for https://github.com/nasa/openmct/issues/5184
|
||||
await page.click('form[name="mctForm"] a:has-text("My Items")');
|
||||
|
||||
await page.locator('text=OK').click();
|
||||
|
||||
// Finish editing and save Telemetry Table
|
||||
await page.locator('.c-button--menu.c-button--major.icon-save').click();
|
||||
@ -114,10 +121,7 @@ test.describe('Move item tests', () => {
|
||||
|
||||
// Continue test regardless of assertion and create it in My Items
|
||||
await page.locator('form[name="mctForm"] >> text=My Items').click();
|
||||
await Promise.all([
|
||||
page.waitForNavigation(),
|
||||
page.locator('text=OK').click()
|
||||
]);
|
||||
await page.locator('text=OK').click();
|
||||
|
||||
// Open My Items
|
||||
await page.locator('text=Open MCT My Items >> span').nth(3).click();
|
||||
|
||||
177
e2e/tests/performance/imagery.perf.spec.js
Normal file
177
e2e/tests/performance/imagery.perf.spec.js
Normal file
@ -0,0 +1,177 @@
|
||||
/*****************************************************************************
|
||||
* Open MCT, Copyright (c) 2014-2022, United States Government
|
||||
* as represented by the Administrator of the National Aeronautics and Space
|
||||
* Administration. All rights reserved.
|
||||
*
|
||||
* Open MCT is licensed under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* http://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* Open MCT includes source code licensed under additional open source
|
||||
* licenses. See the Open Source Licenses file (LICENSES.md) included with
|
||||
* this source code distribution or the Licensing information page available
|
||||
* at runtime from the About dialog for additional information.
|
||||
*****************************************************************************/
|
||||
|
||||
/*
|
||||
This test suite is dedicated to performance tests to ensure that testability of performance
|
||||
is not broken upstream on Open MCT. Any assumptions made downstream will be tested here
|
||||
|
||||
TODO:
|
||||
- Update resolution of performance config
|
||||
- Add Performance Observer on init to push all performance marks
|
||||
- Move client CDP connection to before or to a fixture
|
||||
-
|
||||
|
||||
*/
|
||||
|
||||
const { test, expect } = require('@playwright/test');
|
||||
|
||||
const filePath = 'e2e/test-data/PerformanceDisplayLayout.json';
|
||||
|
||||
test.describe('Performance tests', () => {
|
||||
test.beforeEach(async ({ page, browser }, testInfo) => {
|
||||
// Go to baseURL
|
||||
await page.goto('/', { waitUntil: 'networkidle' });
|
||||
|
||||
// Click a:has-text("My Items")
|
||||
await page.locator('a:has-text("My Items")').click({
|
||||
button: 'right'
|
||||
});
|
||||
|
||||
// Click text=Import from JSON
|
||||
await page.locator('text=Import from JSON').click();
|
||||
|
||||
// Upload Performance Display Layout.json
|
||||
await page.setInputFiles('#fileElem', filePath);
|
||||
|
||||
// Click text=OK
|
||||
await page.locator('text=OK').click();
|
||||
|
||||
await expect(page.locator('a:has-text("Performance Display Layout Display Layout")')).toBeVisible();
|
||||
|
||||
//Create a Chrome Performance Timeline trace to store as a test artifact
|
||||
console.log("\n==== Devtools: startTracing ====\n");
|
||||
await browser.startTracing(page, {
|
||||
path: `${testInfo.outputPath()}-trace.json`,
|
||||
screenshots: true
|
||||
});
|
||||
});
|
||||
test.afterEach(async ({ page, browser}) => {
|
||||
console.log("\n==== Devtools: stopTracing ====\n");
|
||||
await browser.stopTracing();
|
||||
|
||||
/* Measurement Section
|
||||
/ The following section includes a block of performance measurements.
|
||||
*/
|
||||
//Get time difference between viewlarge actionability and evaluate time
|
||||
await page.evaluate(() => (window.performance.measure("machine-time-difference", "viewlarge.start", "viewLarge.start.test")));
|
||||
|
||||
//Get StartTime
|
||||
const startTime = await page.evaluate(() => window.performance.timing.navigationStart);
|
||||
console.log('window.performance.timing.navigationStart', startTime);
|
||||
|
||||
//Get All Performance Marks
|
||||
const getAllMarksJson = await page.evaluate(() => JSON.stringify(window.performance.getEntriesByType("mark")));
|
||||
const getAllMarks = JSON.parse(getAllMarksJson);
|
||||
console.log('window.performance.getEntriesByType("mark")', getAllMarks);
|
||||
|
||||
//Get All Performance Measures
|
||||
const getAllMeasuresJson = await page.evaluate(() => JSON.stringify(window.performance.getEntriesByType("measure")));
|
||||
const getAllMeasures = JSON.parse(getAllMeasuresJson);
|
||||
console.log('window.performance.getEntriesByType("measure")', getAllMeasures);
|
||||
|
||||
});
|
||||
/* The following test will navigate to a previously created Performance Display Layout and measure the
|
||||
/ following metrics:
|
||||
/ - ElementResourceTiming
|
||||
/ - Interaction Timing
|
||||
*/
|
||||
test('Embedded View Large for Imagery is performant in Fixed Time', async ({ page, browser }) => {
|
||||
const client = await page.context().newCDPSession(page);
|
||||
// Tell the DevTools session to record performance metrics
|
||||
// https://chromedevtools.github.io/devtools-protocol/tot/Performance/#method-getMetrics
|
||||
await client.send('Performance.enable');
|
||||
// Go to baseURL
|
||||
await page.goto('/');
|
||||
|
||||
// Search Available after Launch
|
||||
await page.locator('input[type="search"]').click();
|
||||
await page.evaluate(() => window.performance.mark("search-available"));
|
||||
// Fill Search input
|
||||
await page.locator('input[type="search"]').fill('Performance Display Layout');
|
||||
await page.evaluate(() => window.performance.mark("search-entered"));
|
||||
//Search Result Appears and is clicked
|
||||
await Promise.all([
|
||||
page.waitForNavigation(),
|
||||
page.locator('a:has-text("Performance Display Layout")').first().click(),
|
||||
page.evaluate(() => window.performance.mark("click-search-result"))
|
||||
]);
|
||||
|
||||
//Time to Example Imagery Frame loads within Display Layout
|
||||
await page.waitForSelector('.c-imagery__main-image__bg', { state: 'visible'});
|
||||
//Time to Example Imagery object loads
|
||||
await page.waitForSelector('.c-imagery__main-image__background-image', { state: 'visible'});
|
||||
|
||||
//Get background-image url from background-image css prop
|
||||
const backgroundImage = await page.locator('.c-imagery__main-image__background-image');
|
||||
let backgroundImageUrl = await backgroundImage.evaluate((el) => {
|
||||
return window.getComputedStyle(el).getPropertyValue('background-image').match(/url\(([^)]+)\)/)[1];
|
||||
});
|
||||
backgroundImageUrl = backgroundImageUrl.slice(1, -1); //forgive me, padre
|
||||
console.log('backgroundImageurl ' + backgroundImageUrl);
|
||||
|
||||
//Get ResourceTiming of background-image jpg
|
||||
const resourceTimingJson = await page.evaluate((bgImageUrl) =>
|
||||
JSON.stringify(window.performance.getEntriesByName(bgImageUrl).pop()),
|
||||
backgroundImageUrl
|
||||
);
|
||||
console.log('resourceTimingJson ' + resourceTimingJson);
|
||||
|
||||
//Open Large view
|
||||
await page.locator('button:has-text("Large View")').click(); //This action includes the performance.mark named 'viewLarge.start'
|
||||
await page.evaluate(() => window.performance.mark("viewLarge.start.test")); //This is a mark only to compare evaluate timing
|
||||
|
||||
//Time to Imagery Rendered in Large Frame
|
||||
await page.waitForSelector('.c-imagery__main-image__bg', { state: 'visible'});
|
||||
await page.evaluate(() => window.performance.mark("background-image-frame"));
|
||||
|
||||
//Time to Example Imagery object loads
|
||||
await page.waitForSelector('.c-imagery__main-image__background-image', { state: 'visible'});
|
||||
await page.evaluate(() => window.performance.mark("background-image-visible"));
|
||||
|
||||
// Get Current number of images in thumbstrip
|
||||
await page.waitForSelector('.c-imagery__thumb');
|
||||
const thumbCount = await page.locator('.c-imagery__thumb').count();
|
||||
console.log('number of thumbs rendered ' + thumbCount);
|
||||
await page.locator('.c-imagery__thumb').last().click();
|
||||
|
||||
//Get ResourceTiming of all jpg resources
|
||||
const resourceTimingJson2 = await page.evaluate(() =>
|
||||
JSON.stringify(window.performance.getEntriesByType('resource'))
|
||||
);
|
||||
const resourceTiming = JSON.parse(resourceTimingJson2);
|
||||
const jpgResourceTiming = resourceTiming.find((element) =>
|
||||
element.name.includes('.jpg')
|
||||
);
|
||||
console.log('jpgResourceTiming ' + JSON.stringify(jpgResourceTiming));
|
||||
|
||||
// Click Close Icon
|
||||
await page.locator('.c-click-icon').click();
|
||||
await page.evaluate(() => window.performance.mark("view-large-close-button"));
|
||||
|
||||
//await client.send('HeapProfiler.enable');
|
||||
await client.send('HeapProfiler.collectGarbage');
|
||||
|
||||
let performanceMetrics = await client.send('Performance.getMetrics');
|
||||
console.log(performanceMetrics.metrics);
|
||||
|
||||
});
|
||||
});
|
||||
118
e2e/tests/performance/memleak-imagery.perf.spec.js
Normal file
118
e2e/tests/performance/memleak-imagery.perf.spec.js
Normal file
@ -0,0 +1,118 @@
|
||||
/*****************************************************************************
|
||||
* Open MCT, Copyright (c) 2014-2022, United States Government
|
||||
* as represented by the Administrator of the National Aeronautics and Space
|
||||
* Administration. All rights reserved.
|
||||
*
|
||||
* Open MCT is licensed under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* http://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* Open MCT includes source code licensed under additional open source
|
||||
* licenses. See the Open Source Licenses file (LICENSES.md) included with
|
||||
* this source code distribution or the Licensing information page available
|
||||
* at runtime from the About dialog for additional information.
|
||||
*****************************************************************************/
|
||||
|
||||
/*
|
||||
This test suite is an initial example for memory leak testing using performance. This configuration and execution must
|
||||
be kept separate from the traditional performance measurements to avoid any "observer" effects associated with tracing
|
||||
or profiling playwright and/or the browser.
|
||||
|
||||
Based on a pattern identified in https://github.com/trentmwillis/devtools-protocol-demos/blob/master/testing-demos/memory-leak-by-heap.js
|
||||
and https://github.com/paulirish/automated-chrome-profiling/issues/3
|
||||
|
||||
Best path forward: https://github.com/cowchimp/headless-devtools/blob/master/src/Memory/example.js
|
||||
|
||||
*/
|
||||
|
||||
const { test, expect } = require('@playwright/test');
|
||||
|
||||
const filePath = 'e2e/test-data/PerformanceDisplayLayout.json';
|
||||
|
||||
test.describe.skip('Memory Performance tests', () => {
|
||||
test.beforeEach(async ({ page, browser }, testInfo) => {
|
||||
// Go to baseURL
|
||||
await page.goto('/', { waitUntil: 'networkidle' });
|
||||
|
||||
// Click a:has-text("My Items")
|
||||
await page.locator('a:has-text("My Items")').click({
|
||||
button: 'right'
|
||||
});
|
||||
|
||||
// Click text=Import from JSON
|
||||
await page.locator('text=Import from JSON').click();
|
||||
|
||||
// Upload Performance Display Layout.json
|
||||
await page.setInputFiles('#fileElem', filePath);
|
||||
|
||||
// Click text=OK
|
||||
await page.locator('text=OK').click();
|
||||
|
||||
await expect(page.locator('a:has-text("Performance Display Layout Display Layout")')).toBeVisible();
|
||||
});
|
||||
|
||||
test.skip('Embedded View Large for Imagery is performant in Fixed Time', async ({ page, browser }) => {
|
||||
|
||||
await page.goto('/', {waitUntil: 'networkidle'});
|
||||
|
||||
// To to Search Available after Launch
|
||||
await page.locator('input[type="search"]').click();
|
||||
// Fill Search input
|
||||
await page.locator('input[type="search"]').fill('Performance Display Layout');
|
||||
//Search Result Appears and is clicked
|
||||
await Promise.all([
|
||||
page.waitForNavigation(),
|
||||
page.locator('a:has-text("Performance Display Layout")').first().click()
|
||||
]);
|
||||
|
||||
//Time to Example Imagery Frame loads within Display Layout
|
||||
await page.waitForSelector('.c-imagery__main-image__bg', { state: 'visible'});
|
||||
//Time to Example Imagery object loads
|
||||
await page.waitForSelector('.c-imagery__main-image__background-image', { state: 'visible'});
|
||||
|
||||
const client = await page.context().newCDPSession(page);
|
||||
await client.send('HeapProfiler.enable');
|
||||
await client.send('HeapProfiler.startSampling');
|
||||
// await client.send('HeapProfiler.collectGarbage');
|
||||
await client.send('Performance.enable');
|
||||
|
||||
let performanceMetricsBefore = await client.send('Performance.getMetrics');
|
||||
console.log(performanceMetricsBefore.metrics);
|
||||
|
||||
//await client.send('Performance.disable');
|
||||
|
||||
//Open Large view
|
||||
await page.locator('button:has-text("Large View")').click();
|
||||
await client.send('HeapProfiler.takeHeapSnapshot');
|
||||
|
||||
//Time to Imagery Rendered in Large Frame
|
||||
await page.waitForSelector('.c-imagery__main-image__bg', { state: 'visible'});
|
||||
|
||||
//Time to Example Imagery object loads
|
||||
await page.waitForSelector('.c-imagery__main-image__background-image', { state: 'visible'});
|
||||
|
||||
// Click Close Icon
|
||||
await page.locator('.c-click-icon').click();
|
||||
|
||||
//Time to Example Imagery Frame loads within Display Layout
|
||||
await page.waitForSelector('.c-imagery__main-image__bg', { state: 'visible'});
|
||||
//Time to Example Imagery object loads
|
||||
await page.waitForSelector('.c-imagery__main-image__background-image', { state: 'visible'});
|
||||
|
||||
await client.send('HeapProfiler.collectGarbage');
|
||||
//await client.send('Performance.enable');
|
||||
|
||||
let performanceMetricsAfter = await client.send('Performance.getMetrics');
|
||||
console.log(performanceMetricsAfter.metrics);
|
||||
|
||||
//await client.send('Performance.disable');
|
||||
|
||||
});
|
||||
});
|
||||
158
e2e/tests/performance/notebook.perf.spec.js
Normal file
158
e2e/tests/performance/notebook.perf.spec.js
Normal file
@ -0,0 +1,158 @@
|
||||
/*****************************************************************************
|
||||
* Open MCT, Copyright (c) 2014-2022, United States Government
|
||||
* as represented by the Administrator of the National Aeronautics and Space
|
||||
* Administration. All rights reserved.
|
||||
*
|
||||
* Open MCT is licensed under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* http://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* Open MCT includes source code licensed under additional open source
|
||||
* licenses. See the Open Source Licenses file (LICENSES.md) included with
|
||||
* this source code distribution or the Licensing information page available
|
||||
* at runtime from the About dialog for additional information.
|
||||
*****************************************************************************/
|
||||
|
||||
/*
|
||||
This test suite is dedicated to performance tests to ensure that testability of performance
|
||||
is not broken upstream on Open MCT. Any assumptions made downstream will be tested here.
|
||||
|
||||
TODO:
|
||||
- Update resolution of performance config
|
||||
- Add Performance Observer on init to push all performance marks
|
||||
- Move client CDP connection to before or to a fixture
|
||||
|
||||
*/
|
||||
|
||||
const { test, expect } = require('@playwright/test');
|
||||
|
||||
const notebookFilePath = 'e2e/test-data/PerformanceNotebook.json';
|
||||
|
||||
test.describe('Performance tests', () => {
|
||||
test.beforeEach(async ({ page, browser }, testInfo) => {
|
||||
// Go to baseURL
|
||||
await page.goto('/', { waitUntil: 'networkidle' });
|
||||
|
||||
// Click a:has-text("My Items")
|
||||
await page.locator('a:has-text("My Items")').click({
|
||||
button: 'right'
|
||||
});
|
||||
|
||||
// Click text=Import from JSON
|
||||
await page.locator('text=Import from JSON').click();
|
||||
|
||||
// Upload Performance Display Layout.json
|
||||
await page.setInputFiles('#fileElem', notebookFilePath);
|
||||
|
||||
// TODO Fix this
|
||||
await page.locator('text=OK >> nth=1').click();
|
||||
|
||||
await expect(page.locator('a:has-text("Performance Notebook")')).toBeVisible();
|
||||
|
||||
//Create a Chrome Performance Timeline trace to store as a test artifact
|
||||
console.log("\n==== Devtools: startTracing ====\n");
|
||||
await browser.startTracing(page, {
|
||||
path: `${testInfo.outputPath()}-trace.json`,
|
||||
screenshots: true
|
||||
});
|
||||
});
|
||||
test.afterEach(async ({ page, browser}) => {
|
||||
console.log("\n==== Devtools: stopTracing ====\n");
|
||||
await browser.stopTracing();
|
||||
|
||||
/* Measurement Section
|
||||
/ The following section includes a block of performance measurements.
|
||||
*/
|
||||
const startTime = await page.evaluate(() => window.performance.timing.navigationStart);
|
||||
console.log('window.performance.timing.navigationStart', startTime);
|
||||
|
||||
//Get All Performance Marks
|
||||
const getAllMarksJson = await page.evaluate(() => JSON.stringify(window.performance.getEntriesByType("mark")));
|
||||
const getAllMarks = JSON.parse(getAllMarksJson);
|
||||
console.log('window.performance.getEntriesByType("mark")', getAllMarks);
|
||||
|
||||
//Get All Performance Measures
|
||||
const getAllMeasuresJson = await page.evaluate(() => JSON.stringify(window.performance.getEntriesByType("measure")));
|
||||
const getAllMeasures = JSON.parse(getAllMeasuresJson);
|
||||
console.log('window.performance.getEntriesByType("measure")', getAllMeasures);
|
||||
|
||||
});
|
||||
/* The following test will navigate to a previously created Performance Display Layout and measure the
|
||||
/ following metrics:
|
||||
/ - ElementResourceTiming
|
||||
/ - Interaction Timing
|
||||
*/
|
||||
test('Notebook Search, Add Entry, Update Entry are performant', async ({ page, browser }) => {
|
||||
const client = await page.context().newCDPSession(page);
|
||||
// Tell the DevTools session to record performance metrics
|
||||
// https://chromedevtools.github.io/devtools-protocol/tot/Performance/#method-getMetrics
|
||||
await client.send('Performance.enable');
|
||||
// Go to baseURL
|
||||
await page.goto('/');
|
||||
|
||||
// To to Search Available after Launch
|
||||
await page.locator('input[type="search"]').click();
|
||||
await page.evaluate(() => window.performance.mark("search-available"));
|
||||
// Fill Search input
|
||||
await page.locator('input[type="search"]').fill('Performance Notebook');
|
||||
await page.evaluate(() => window.performance.mark("search-entered"));
|
||||
//Search Result Appears and is clicked
|
||||
await Promise.all([
|
||||
page.waitForNavigation(),
|
||||
page.locator('a:has-text("Performance Notebook")').first().click(),
|
||||
page.evaluate(() => window.performance.mark("click-search-result"))
|
||||
]);
|
||||
|
||||
await page.waitForSelector('.c-tree__item c-tree-and-search__loading loading', {state: 'hidden'});
|
||||
await page.evaluate(() => window.performance.mark("search-spinner-gone"));
|
||||
|
||||
await page.waitForSelector('.l-browse-bar__object-name', { state: 'visible'});
|
||||
await page.evaluate(() => window.performance.mark("object-title-appears"));
|
||||
|
||||
await page.waitForSelector('.c-notebook__entry >> nth=0', { state: 'visible'});
|
||||
await page.evaluate(() => window.performance.mark("notebook-entry-appears"));
|
||||
|
||||
// Click Add new Notebook Entry
|
||||
await page.locator('.c-notebook__drag-area').click();
|
||||
await page.evaluate(() => window.performance.mark("new-notebook-entry-created"));
|
||||
|
||||
// Enter Notebook Entry text
|
||||
await page.locator('div.c-ne__text').last().fill('New Entry');
|
||||
await page.keyboard.press('Enter');
|
||||
await page.evaluate(() => window.performance.mark("new-notebook-entry-filled"));
|
||||
|
||||
//Individual Notebook Entry Search
|
||||
await page.evaluate(() => window.performance.mark("notebook-search-start"));
|
||||
await page.locator('.c-notebook__search >> input').fill('Existing Entry');
|
||||
await page.evaluate(() => window.performance.mark("notebook-search-filled"));
|
||||
await page.waitForSelector('text=Search Results (3)', { state: 'visible'});
|
||||
await page.evaluate(() => window.performance.mark("notebook-search-processed"));
|
||||
await page.waitForSelector('.c-notebook__entry >> nth=2', { state: 'visible'});
|
||||
await page.evaluate(() => window.performance.mark("notebook-search-processed"));
|
||||
|
||||
//Clear Search
|
||||
await page.locator('.c-search.c-notebook__search .c-search__clear-input').click();
|
||||
await page.evaluate(() => window.performance.mark("notebook-search-processed"));
|
||||
|
||||
// Hover on Last
|
||||
await page.evaluate(() => window.performance.mark("new-notebook-entry-delete"));
|
||||
await page.locator('div.c-ne__time-and-content').last().hover();
|
||||
await page.locator('button[title="Delete this entry"]').last().click();
|
||||
await page.locator('button:has-text("Ok")').click();
|
||||
await page.waitForSelector('.c-notebook__entry >> nth=3', { state: 'detached'});
|
||||
await page.evaluate(() => window.performance.mark("new-notebook-entry-deleted"));
|
||||
|
||||
//await client.send('HeapProfiler.enable');
|
||||
await client.send('HeapProfiler.collectGarbage');
|
||||
|
||||
let performanceMetrics = await client.send('Performance.getMetrics');
|
||||
console.log(performanceMetrics.metrics);
|
||||
});
|
||||
});
|
||||
@ -42,6 +42,9 @@ test('Create new Condition Set object and store @localStorage', async ({ page, c
|
||||
// Click text=Condition Set
|
||||
await page.click('text=Condition Set');
|
||||
|
||||
// Click on My Items in Tree. Workaround for https://github.com/nasa/openmct/issues/5184
|
||||
await page.click('form[name="mctForm"] a:has-text("My Items")');
|
||||
|
||||
// Click text=OK
|
||||
await Promise.all([
|
||||
page.waitForNavigation(),
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 15 KiB After Width: | Height: | Size: 18 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 18 KiB After Width: | Height: | Size: 19 KiB |
@ -6,14 +6,13 @@
|
||||
"@babel/eslint-parser": "7.16.3",
|
||||
"@braintree/sanitize-url": "6.0.0",
|
||||
"@percy/cli": "1.2.1",
|
||||
"@percy/playwright": "1.0.3",
|
||||
"@percy/playwright": "1.0.4",
|
||||
"@playwright/test": "1.21.1",
|
||||
"@types/eventemitter3": "^1.0.0",
|
||||
"@types/jasmine": "^4.0.1",
|
||||
"@types/karma": "^6.3.2",
|
||||
"@types/lodash": "^4.14.178",
|
||||
"@types/mocha": "^9.1.0",
|
||||
"allure-playwright": "2.0.0-beta.15",
|
||||
"babel-loader": "8.2.3",
|
||||
"babel-plugin-istanbul": "6.1.1",
|
||||
"comma-separated-values": "3.6.4",
|
||||
@ -92,11 +91,12 @@
|
||||
"test": "cross-env NODE_OPTIONS=\"--max_old_space_size=4096\" karma start --single-run",
|
||||
"test:firefox": "cross-env NODE_OPTIONS=\"--max_old_space_size=4096\" karma start --single-run --browsers=FirefoxHeadless",
|
||||
"test:debug": "cross-env NODE_ENV=debug karma start --no-single-run",
|
||||
"test:e2e:ci": "npx playwright test --config=e2e/playwright-ci.config.js --project=chrome smoke default condition timeConductor branding clock exampleImagery",
|
||||
"test:e2e:ci": "npx playwright test --config=e2e/playwright-ci.config.js --project=chrome smoke branding default condition timeConductor clock exampleImagery notebook persistence performance",
|
||||
"test:e2e:local": "npx playwright test --config=e2e/playwright-local.config.js --project=chrome",
|
||||
"test:e2e:updatesnapshots": "npx playwright test --config=e2e/playwright-local.config.js --project=chrome --grep @snapshot --update-snapshots",
|
||||
"test:e2e:visual": "percy exec --config ./e2e/.percy.yml -- npx playwright test --config=e2e/playwright-visual.config.js",
|
||||
"test:e2e:full": "npx playwright test --config=e2e/playwright-ci.config.js",
|
||||
"test:perf": "npx playwright test --config=e2e/playwright-performance.config.js",
|
||||
"test:watch": "cross-env NODE_OPTIONS=\"--max_old_space_size=4096\" karma start --no-single-run",
|
||||
"jsdoc": "jsdoc -c jsdoc.json -R API.md -r -d dist/docs/api",
|
||||
"update-about-dialog-copyright": "perl -pi -e 's/20\\d\\d\\-202\\d/2014\\-2022/gm' ./src/ui/layout/AboutDialog.vue",
|
||||
|
||||
Loading…
Reference in New Issue
Block a user