ExternalVendorCode/openmct
1
0
Fork 0
mirror of https://github.com/nasa/openmct.git synced 2024-12-18 20:57:53 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

move performance tests to GHA (#7412)

Browse Source 
* move performance tests to GHA

* no need for chrome beta

* Add baseline imagery test

* skip flaky app

* lint
This commit is contained in:
John Hill 2024-01-24 21:26:48 -08:00 committed by GitHub
parent 0340fe18fa
commit 450cab428f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 217 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
.circleci/config.yml
View File

@ -5,20 +5,20 @@ executors:
- image: mcr.microsoft.com/playwright:v1.39.0-focal - image: mcr.microsoft.com/playwright:v1.39.0-focal
environment: environment:
NODE_ENV: development # Needed to ensure 'dist' folder created and devDependencies installed NODE_ENV: development # Needed to ensure 'dist' folder created and devDependencies installed
PERCY_POSTINSTALL_BROWSER: 'true' # Needed to store the percy browser in cache deps PERCY_POSTINSTALL_BROWSER: "true" # Needed to store the percy browser in cache deps
PERCY_LOGLEVEL: 'debug' # Enable DEBUG level logging for Percy (Issue: https://github.com/nasa/openmct/issues/5742) PERCY_LOGLEVEL: "debug" # Enable DEBUG level logging for Percy (Issue: https://github.com/nasa/openmct/issues/5742)
ubuntu: ubuntu:
machine: machine:
image: ubuntu-2204:current image: ubuntu-2204:current
docker_layer_caching: true docker_layer_caching: true
parameters: parameters:
BUST_CACHE: BUST_CACHE:
description: 'Set this with the CircleCI UI Trigger Workflow button (boolean = true) to bust the cache!' description: "Set this with the CircleCI UI Trigger Workflow button (boolean = true) to bust the cache!"
default: false default: false
type: boolean type: boolean
commands: commands:
build_and_install: build_and_install:
description: 'All steps used to build and install. Will use cache if found' description: "All steps used to build and install. Will use cache if found"
parameters: parameters:
node-version: node-version:
type: string type: string
@ -30,7 +30,7 @@ commands:
node-version: << parameters.node-version >> node-version: << parameters.node-version >>
- run: npm install --no-audit --progress=false - run: npm install --no-audit --progress=false
restore_cache_cmd: restore_cache_cmd:
description: 'Custom command for restoring cache with the ability to bust cache. When BUST_CACHE is set to true, jobs will not restore cache' description: "Custom command for restoring cache with the ability to bust cache. When BUST_CACHE is set to true, jobs will not restore cache"
parameters: parameters:
node-version: node-version:
type: string type: string
@ -42,7 +42,7 @@ commands:
- restore_cache: - restore_cache:
key: deps--{{ arch }}--{{ .Branch }}--<< parameters.node-version >>--{{ checksum "package.json" }}-{{ checksum ".circleci/config.yml" }} key: deps--{{ arch }}--{{ .Branch }}--<< parameters.node-version >>--{{ checksum "package.json" }}-{{ checksum ".circleci/config.yml" }}
save_cache_cmd: save_cache_cmd:
description: 'Custom command for saving cache.' description: "Custom command for saving cache."
parameters: parameters:
node-version: node-version:
type: string type: string
@ -53,7 +53,7 @@ commands:
- ~/.npm - ~/.npm
- node_modules - node_modules
generate_and_store_version_and_filesystem_artifacts: generate_and_store_version_and_filesystem_artifacts:
description: 'Track important packages and files' description: "Track important packages and files"
steps: steps:
- run: | - run: |
[[ $EUID -ne 0 ]] && (sudo mkdir -p /tmp/artifacts && sudo chmod 777 /tmp/artifacts) || (mkdir -p /tmp/artifacts && chmod 777 /tmp/artifacts) [[ $EUID -ne 0 ]] && (sudo mkdir -p /tmp/artifacts && sudo chmod 777 /tmp/artifacts) || (mkdir -p /tmp/artifacts && chmod 777 /tmp/artifacts)
@ -64,7 +64,7 @@ commands:
- store_artifacts: - store_artifacts:
path: /tmp/artifacts/ path: /tmp/artifacts/
generate_e2e_code_cov_report: generate_e2e_code_cov_report:
description: 'Generate e2e code coverage artifacts and publish to codecov.io. Needed to that we can ignore the exit code status of the npm run test' description: "Generate e2e code coverage artifacts and publish to codecov.io. Needed to that we can ignore the exit code status of the npm run test"
parameters: parameters:
suite: suite:
type: string type: string
@ -129,7 +129,7 @@ jobs:
node-version: lts/hydrogen node-version: lts/hydrogen
- when: #Only install chrome-beta when running the 'full' suite to save $$$ - when: #Only install chrome-beta when running the 'full' suite to save $$$
condition: condition:
equal: ['full', <<parameters.suite>>] equal: ["full", <<parameters.suite>>]
steps: steps:
- run: npx playwright install chrome-beta - run: npx playwright install chrome-beta
- run: SHARD="$((${CIRCLE_NODE_INDEX}+1))"; npm run test:e2e:<<parameters.suite>> -- --shard=${SHARD}/${CIRCLE_NODE_TOTAL} - run: SHARD="$((${CIRCLE_NODE_INDEX}+1))"; npm run test:e2e:<<parameters.suite>> -- --shard=${SHARD}/${CIRCLE_NODE_TOTAL}
@ -251,8 +251,6 @@ workflows:
- e2e-test: - e2e-test:
name: e2e-stable name: e2e-stable
suite: stable suite: stable
- mem-test
- perf-test
- visual-a11y-tests: - visual-a11y-tests:
name: visual-test-ci name: visual-test-ci
suite: ci suite: ci
@ -278,7 +276,7 @@ workflows:
- e2e-couchdb - e2e-couchdb
triggers: triggers:
- schedule: - schedule:
cron: '0 0 * * *' cron: "0 0 * * *"
filters: filters:
branches: branches:
only: only:

58
.github/workflows/e2e-perf.yml vendored Normal file
View File

@ -0,0 +1,58 @@
name: 'e2e-perf'
on:
push:
branches: master
workflow_dispatch:
pull_request:
types:
- labeled
- opened
schedule:
- cron: '0 0 * * *'
jobs:
e2e-full:
if: contains(github.event.pull_request.labels.*.name, 'pr:e2e:perf') || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 'lts/hydrogen'
- name: Cache NPM dependencies
uses: actions/cache@v3
with:
path: ~/.npm
key: ${{ runner.os }}-node-${{ hashFiles('**/package.json') }}
restore-keys: |
${{ runner.os }}-node-
- run: npx playwright@1.39.0 install
- run: npm install --cache ~/.npm --no-audit --progress=false
- run: npm run test:perf:localhost
- run: npm run test:perf:contract
- run: npm run test:perf:memory
- name: Archive test results
if: success() || failure()
uses: actions/upload-artifact@v3
with:
path: test-results
- name: Remove pr:e2e:perf label (if present)
if: always()
uses: actions/github-script@v6
with:
script: |
const { owner, repo, number } = context.issue;
const labelToRemove = 'pr:e2e:perf';
try {
await github.rest.issues.removeLabel({
owner,
repo,
issue_number: number,
name: labelToRemove
});
} catch (error) {
core.warning(`Failed to remove ' + labelToRemove + ' label: ${error.message}`);
}

110
e2e/tests/functional/plugins/plot/overlayPlot.e2e.spec.js
View File

@ -63,78 +63,80 @@ test.describe('Overlay Plot', () => {
await expect(seriesColorSwatch).toHaveCSS('background-color', 'rgb(255, 166, 61)'); await expect(seriesColorSwatch).toHaveCSS('background-color', 'rgb(255, 166, 61)');
}); });
test('Limit lines persist when series is moved to another Y Axis and on refresh', async ({ //skipping due to https://github.com/nasa/openmct/issues/7405
page test.fixme(
}) => { 'Limit lines persist when series is moved to another Y Axis and on refresh',
test.info().annotations.push({ async ({ page }) => {
type: 'issue', test.info().annotations.push({
description: 'https://github.com/nasa/openmct/issues/6338' type: 'issue',
}); description: 'https://github.com/nasa/openmct/issues/6338'
// Create an Overlay Plot with a default SWG });
const overlayPlot = await createDomainObjectWithDefaults(page, { // Create an Overlay Plot with a default SWG
type: 'Overlay Plot' const overlayPlot = await createDomainObjectWithDefaults(page, {
}); type: 'Overlay Plot'
});
const swgA = await createDomainObjectWithDefaults(page, { const swgA = await createDomainObjectWithDefaults(page, {
type: 'Sine Wave Generator', type: 'Sine Wave Generator',
parent: overlayPlot.uuid parent: overlayPlot.uuid
}); });
await page.goto(overlayPlot.url); await page.goto(overlayPlot.url);
// Assert that no limit lines are shown by default // Assert that no limit lines are shown by default
await page.waitForSelector('.js-limit-area', { state: 'attached' }); await page.waitForSelector('.js-limit-area', { state: 'attached' });
expect(await page.locator('.c-plot-limit-line').count()).toBe(0); expect(await page.locator('.c-plot-limit-line').count()).toBe(0);
// Enter edit mode // Enter edit mode
await page.getByLabel('Edit Object').click(); await page.getByLabel('Edit Object').click();
// Expand the "Sine Wave Generator" plot series options and enable limit lines // Expand the "Sine Wave Generator" plot series options and enable limit lines
await page.getByRole('tab', { name: 'Config' }).click(); await page.getByRole('tab', { name: 'Config' }).click();
await page await page
.getByRole('list', { name: 'Plot Series Properties' }) .getByRole('list', { name: 'Plot Series Properties' })
.locator('span') .locator('span')
.first() .first()
.click(); .click();
await page await page
.getByRole('list', { name: 'Plot Series Properties' }) .getByRole('list', { name: 'Plot Series Properties' })
.locator('[title="Display limit lines"]~div input') .locator('[title="Display limit lines"]~div input')
.check(); .check();
await assertLimitLinesExistAndAreVisible(page); await assertLimitLinesExistAndAreVisible(page);
// Save (exit edit mode) // Save (exit edit mode)
await page.locator('button[title="Save"]').click(); await page.locator('button[title="Save"]').click();
await page.getByRole('listitem', { name: 'Save and Finish Editing' }).click(); await page.getByRole('listitem', { name: 'Save and Finish Editing' }).click();
await assertLimitLinesExistAndAreVisible(page); await assertLimitLinesExistAndAreVisible(page);
await page.reload(); await page.reload();
await assertLimitLinesExistAndAreVisible(page); await assertLimitLinesExistAndAreVisible(page);
// Enter edit mode // Enter edit mode
await page.getByLabel('Edit Object').click(); await page.getByLabel('Edit Object').click();
await page.getByRole('tab', { name: 'Elements' }).click(); await page.getByRole('tab', { name: 'Elements' }).click();
// Drag Sine Wave Generator series from Y Axis 1 into Y Axis 2 // Drag Sine Wave Generator series from Y Axis 1 into Y Axis 2
await page await page
.locator(`#inspector-elements-tree >> text=${swgA.name}`) .locator(`#inspector-elements-tree >> text=${swgA.name}`)
.dragTo(page.locator('[aria-label="Element Item Group Y Axis 2"]')); .dragTo(page.locator('[aria-label="Element Item Group Y Axis 2"]'));
await assertLimitLinesExistAndAreVisible(page); await assertLimitLinesExistAndAreVisible(page);
// Save (exit edit mode) // Save (exit edit mode)
await page.locator('button[title="Save"]').click(); await page.locator('button[title="Save"]').click();
await page.getByRole('listitem', { name: 'Save and Finish Editing' }).click(); await page.getByRole('listitem', { name: 'Save and Finish Editing' }).click();
await assertLimitLinesExistAndAreVisible(page); await assertLimitLinesExistAndAreVisible(page);
await page.reload(); await page.reload();
await assertLimitLinesExistAndAreVisible(page); await assertLimitLinesExistAndAreVisible(page);
}); }
);
test('The elements pool supports dragging series into multiple y-axis buckets', async ({ test('The elements pool supports dragging series into multiple y-axis buckets', async ({
page page

93
e2e/tests/visual-a11y/imagery.visual.spec.js Normal file
View File

@ -0,0 +1,93 @@
/*****************************************************************************
* Open MCT, Copyright (c) 2014-2024, United States Government
* as represented by the Administrator of the National Aeronautics and Space
* Administration. All rights reserved.
*
* Open MCT is licensed under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0.
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*
* Open MCT includes source code licensed under additional open source
* licenses. See the Open Source Licenses file (LICENSES.md) included with
* this source code distribution or the Licensing information page available
* at runtime from the About dialog for additional information.
*****************************************************************************/
import percySnapshot from '@percy/playwright';
import { createDomainObjectWithDefaults, setRealTimeMode } from '../../appActions.js';
import { VISUAL_URL } from '../../constants.js';
import { expect, test } from '../../pluginFixtures.js';
test.describe('Visual - Example Imagery', () => {
let exampleImagery;
let parentLayout;
test.beforeEach(async ({ page }) => {
await page.goto(VISUAL_URL, { waitUntil: 'domcontentloaded' });
parentLayout = await createDomainObjectWithDefaults(page, {
type: 'Display Layout',
name: 'Parent Layout'
});
exampleImagery = await createDomainObjectWithDefaults(page, {
type: 'Example Imagery',
name: 'Example Imagery Test',
parent: parentLayout.uuid
});
// Modify Example Imagery to create a really stable Example Imagery
await page.goto(exampleImagery.url, { waitUntil: 'domcontentloaded' });
await page.getByRole('button', { name: 'More actions' }).click();
await page.getByRole('menuitem', { name: 'Edit Properties...' }).click();
await page
.locator('#imageLocation-textarea')
.fill(
'https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg'
);
await page.getByRole('button', { name: 'Save' }).click();
await page.reload({ waitUntil: 'domcontentloaded' });
await page.getByTitle('Collapse Browse Pane').click();
await page.getByTitle('Collapse Inspect Pane').click();
});
test('Example Imagery in Fixed Time', async ({ page, theme }) => {
await page.goto(exampleImagery.url, { waitUntil: 'domcontentloaded' });
await expect(page.getByLabel('Image Wrapper')).toBeVisible();
await percySnapshot(page, `Example Imagery in Fixed Time (theme: ${theme})`);
await page.getByLabel('Image Wrapper').hover();
await percySnapshot(page, `Example Imagery Hover in Fixed Time (theme: ${theme})`);
});
test('Example Imagery in Real Time', async ({ page, theme }) => {
await page.goto(exampleImagery.url, { waitUntil: 'domcontentloaded' });
await setRealTimeMode(page, true);
//Temporary to close the dialog
await page.getByLabel('Submit time offsets').click();
await expect(page.getByLabel('Image Wrapper')).toBeVisible();
await percySnapshot(page, `Example Imagery in Real Time (theme: ${theme})`);
});
test('Example Imagery in Display Layout', async ({ page, theme }) => {
await page.goto(parentLayout.url, { waitUntil: 'domcontentloaded' });
await expect(page.getByLabel('Image Wrapper')).toBeVisible();
await percySnapshot(page, `Example Imagery in Display Layout (theme: ${theme})`);
});
});