openmct/.circleci/config.yml
John Hill 0413e77d8a
test(e2e): Major refactor and stabilization of e2e tests (#7581)
* fix: update broken locator

* update eslint package

* first pass of lint fixes

* update package

* change ruleset

* update component tests to match linting rules

* driveby

* start to factor out bad locators

* update gauge component

* update notebook snapshot drop area

* Update plot aria

* add draggable true to tree items

* update package

* driveby to remove dead code

* unneeded

* unneeded

* tells a screenreader that this is a row and a cell

* adds an id for dragondrops

* this should be a button

* first pass at fixing tooltip selectors

* review comments

* Updating more tests

* update to remove expect expect given our use of check functions

* add expand component

* move role around

* update more locators

* force

* new local storage

* remove choochoo steps

* test: do `lint:fix` and also add back accidentally removed code

* test: add back more removed code

* test: remove `unstable` annotation from tests which are not unstable

* test: remove invalid test-- the "new" time conductor doesn't allow for millisecond changes in fixed time

* test: fix unstable gauge test

* test: remove useless asserts-- this was secretly non-functional. now that we've fixed it, it makes no sense and just fails

* test: add back accidentally removed changes

* test: revert changes that break test

* test: more fixes

* Remove all notion of the unstable/stable e2e tests

* test: eviscerate the flake with FACTS and LOGIC

* test: fix anotha one

* lint fixes

* test: no need to wait for save dialog

* test: fix more tests

* lint: fix more warnings

* test: fix anotha one

* test: use `toHaveLength` instead of `.length).toBe()`

* test: stabilize tabs view example imagery test

* fix: more tests be fixed

* test: more `toHaveCount()`s please

* test: revert more accidentally removed fixes

* test: fix selector

* test: fix anotha one

* update lint rules to clean up bad locators in shared fixtures

* update and remove bad appActions

* test: fix some restricted notebook tests

* test: mass find/replace to enforce `toHaveCount()` instead of `.count()).toBe()`

* Remove some bad appActions and update text

* test: fix da tree tests

* test: await not await await

* test: fix upload plan appAction and add a11y

* Updating externalFixtures with best practice locators and add missing appAction framework tests

* test: fix test

* test: fix appAction test for plans

* test: yum yum fix'em up and get rid of some dragon drops

* fix: alas, a `.only()` got my hopes up that i was done fixing tests

* test: add `setTimeConductorMode` test "suite" which covers most TC related appActions

* test: fix arg

* test(couchdb): fix some network tests via expect polling

* Stabalize visual test

* getCanasPixels

* test: stabilize tooltip telemetry table test, better a11y for tooltips

* chore: update to use `docker compose` instead of `docker-compose`

* New rules, new tests, new me

* fix sort order

* test: add `waitForPlotsToRender` framework test, passthru timeout override

* test: remove `clockOptions` test as we have `page.clock` now

* test: refactor out `overrideClock`

* test: use `clock.install` instead

* test: use `clock.install` instead

* time clock fix

* test: fix timer tests

* remove ever reference to old base fixture

* test: stabilize restricted notebook test

* lint fixes

* test: use clock.install

* update timelist

* test: update visual tests to use `page.clock()`, update snapshots

* test: stabilize tree renaming/reordering test

* a11y: add aria-label and role=region to object view

* refactor: use `dragTo`

* refactor: use `dragTo`, other small fixes

* test: use `page.clock()` to stabilize tooltip telemetry table test

* test: use web-first assertion to stabilize staleness test

* test: knock out a few more `page.click`s

* test: destroy all `page.click()`s

* refactor: consistently use `'Ok'` instead of `'OK'` and `'Ok'` mixed

* test: remove gauge aria label

* test: more test fixes

* test: more fixes and refactors

* docs: add comment

* test: refactor all instances of `dragAndDrop`

* test: remove redundant test (covered in previous test steps)

* test: stabilize imagery operations tests for display layout

* chore: remove bad unicorn rule

* chore(lint): remove unused disable directives

---------

Co-authored-by: Jesse Mazzella <jesse.d.mazzella@nasa.gov>
2024-08-07 21:36:14 +00:00

290 lines
10 KiB
YAML

version: 2.1
orbs:
node: circleci/node@5.2.0
browser-tools: circleci/browser-tools@1.3.0
executors:
pw-focal-development:
docker:
- image: mcr.microsoft.com/playwright:v1.45.2-focal
environment:
NODE_ENV: development # Needed to ensure 'dist' folder created and devDependencies installed
PERCY_POSTINSTALL_BROWSER: 'true' # Needed to store the percy browser in cache deps
PERCY_LOGLEVEL: 'debug' # Enable DEBUG level logging for Percy (Issue: https://github.com/nasa/openmct/issues/5742)
PERCY_PARALLEL_TOTAL: 2
ubuntu:
machine:
image: ubuntu-2204:current
docker_layer_caching: true
commands:
build_and_install:
description: 'All steps used to build and install.'
parameters:
node-version:
type: string
steps:
- checkout
- node/install:
node-version: << parameters.node-version >>
- node/install-packages
generate_and_store_version_and_filesystem_artifacts:
description: 'Track important packages and files'
steps:
- run: |
[[ $EUID -ne 0 ]] && (sudo mkdir -p /tmp/artifacts && sudo chmod 777 /tmp/artifacts) || (mkdir -p /tmp/artifacts && chmod 777 /tmp/artifacts)
printenv NODE_ENV >> /tmp/artifacts/NODE_ENV.txt || true
npm -v >> /tmp/artifacts/npm-version.txt
node -v >> /tmp/artifacts/node-version.txt
ls -latR >> /tmp/artifacts/dir.txt
- store_artifacts:
path: /tmp/artifacts/
generate_e2e_code_cov_report:
description: 'Generate e2e code coverage artifacts and publish to codecov.io. Needed to that we can ignore the exit code status of the npm run test'
parameters:
suite:
type: string
steps:
- run: npm run cov:e2e:report || true
- run: npm run cov:e2e:<<parameters.suite>>:publish
jobs:
npm-audit:
parameters:
node-version:
type: string
executor: pw-focal-development
steps:
- build_and_install:
node-version: <<parameters.node-version>>
- run: npm audit --audit-level=low
- generate_and_store_version_and_filesystem_artifacts
lint:
parameters:
node-version:
type: string
executor: pw-focal-development
steps:
- build_and_install:
node-version: <<parameters.node-version>>
- run: npm run lint
- generate_and_store_version_and_filesystem_artifacts
unit-test:
parameters:
node-version:
type: string
executor: pw-focal-development
steps:
- build_and_install:
node-version: <<parameters.node-version>>
- browser-tools/install-chrome:
replace-existing: false
- run:
command: |
mkdir -p dist/reports/tests/
TESTFILES=$(circleci tests glob "src/**/*Spec.js")
echo "$TESTFILES" | circleci tests run --command="xargs npm run test" --verbose
- run: npm run cov:unit:publish
- store_test_results:
path: dist/reports/tests/
- store_artifacts:
path: coverage
- when:
condition:
equal: [42, 42] # Always generate version artifacts regardless of test failure https://discuss.circleci.com/t/make-custom-command-run-always-with-when-always/38957/2
steps:
- generate_and_store_version_and_filesystem_artifacts
e2e-test:
parameters:
suite: #ci or full
type: string
executor: pw-focal-development
parallelism: 7
steps:
- build_and_install:
node-version: lts/hydrogen
- when: #Only install chrome-beta when running the 'full' suite to save $$$
condition:
equal: ['full', <<parameters.suite>>]
steps:
- run: npx playwright install chrome-beta
- run:
command: |
mkdir test-results
TESTFILES=$(circleci tests glob "e2e/**/*.spec.js")
echo "$TESTFILES" | circleci tests run --command="xargs npm run test:e2e:<<parameters.suite>>" --verbose --split-by=timings
- when:
condition:
equal: [42, 42] # Always run codecov reports regardless of test failure https://discuss.circleci.com/t/make-custom-command-run-always-with-when-always/38957/2
steps:
- generate_e2e_code_cov_report:
suite: <<parameters.suite>>
- store_test_results:
path: test-results/results.xml
- store_artifacts:
path: test-results
- store_artifacts:
path: coverage
- store_artifacts:
path: html-test-results
- when:
condition:
equal: [42, 42] # Always generate version artifacts regardless of test failure https://discuss.circleci.com/t/make-custom-command-run-always-with-when-always/38957/2
steps:
- generate_and_store_version_and_filesystem_artifacts
e2e-mobile:
executor: pw-focal-development
steps:
- build_and_install:
node-version: lts/hydrogen
- run: npm run test:e2e:mobile
- when:
condition:
equal: [42, 42] # Always run codecov reports regardless of test failure https://discuss.circleci.com/t/make-custom-command-run-always-with-when-always/38957/2
steps:
- generate_e2e_code_cov_report:
suite: full
- store_test_results:
path: test-results/results.xml
- store_artifacts:
path: test-results
- store_artifacts:
path: coverage
- store_artifacts:
path: html-test-results
- when:
condition:
equal: [42, 42] # Always generate version artifacts regardless of test failure https://discuss.circleci.com/t/make-custom-command-run-always-with-when-always/38957/2
steps:
- generate_and_store_version_and_filesystem_artifacts
e2e-couchdb:
executor: ubuntu
steps:
- build_and_install:
node-version: lts/hydrogen
- run: npx playwright@1.45.2 install #Necessary for bare ubuntu machine
- run: |
export $(cat src/plugins/persistence/couch/.env.ci | xargs)
docker compose -f src/plugins/persistence/couch/couchdb-compose.yaml up --detach
sleep 3
bash src/plugins/persistence/couch/setup-couchdb.sh
- run: sh src/plugins/persistence/couch/replace-localstorage-with-couchdb-indexhtml.sh #Replace LocalStorage Plugin with CouchDB
- run: npm run test:e2e:couchdb
- when:
condition:
equal: [42, 42] # Always run codecov reports regardless of test failure https://discuss.circleci.com/t/make-custom-command-run-always-with-when-always/38957/2
steps:
- generate_e2e_code_cov_report:
suite: full #add to full suite
- store_test_results:
path: test-results/results.xml
- store_artifacts:
path: test-results
- store_artifacts:
path: coverage
- store_artifacts:
path: html-test-results
- when:
condition:
equal: [42, 42] # Always generate version artifacts regardless of test failure https://discuss.circleci.com/t/make-custom-command-run-always-with-when-always/38957/2
steps:
- generate_and_store_version_and_filesystem_artifacts
mem-test:
executor: pw-focal-development
steps:
- build_and_install:
node-version: lts/hydrogen
- run: npm run test:perf:memory
- store_test_results:
path: test-results/results.xml
- store_artifacts:
path: test-results
- store_artifacts:
path: html-test-results
- when:
condition:
equal: [42, 42] # Always run codecov reports regardless of test failure https://discuss.circleci.com/t/make-custom-command-run-always-with-when-always/38957/2
steps:
- generate_and_store_version_and_filesystem_artifacts
perf-test:
executor: pw-focal-development
steps:
- build_and_install:
node-version: lts/hydrogen
- run: npm run test:perf:localhost
- run: npm run test:perf:contract
- store_test_results:
path: test-results/results.xml
- store_artifacts:
path: test-results
- store_artifacts:
path: html-test-results
- when:
condition:
equal: [42, 42] # Always run codecov reports regardless of test failure https://discuss.circleci.com/t/make-custom-command-run-always-with-when-always/38957/2
steps:
- generate_and_store_version_and_filesystem_artifacts
visual-a11y:
parameters:
suite:
type: string # ci or full
executor: pw-focal-development
parallelism: 2
steps:
- build_and_install:
node-version: lts/iron
- run: SHARD="$((${CIRCLE_NODE_INDEX}+1))"; npm run test:e2e:visual:<<parameters.suite>> -- --shard=${SHARD}/${CIRCLE_NODE_TOTAL}
- store_test_results:
path: test-results/results.xml
- store_artifacts:
path: test-results
- store_artifacts:
path: html-test-results
- when:
condition:
equal: [42, 42] # Always generate version artifacts regardless of test failure https://discuss.circleci.com/t/make-custom-command-run-always-with-when-always/38957/2
steps:
- generate_and_store_version_and_filesystem_artifacts
workflows:
overall-circleci-commit-status: #These jobs run on every commit
jobs:
- lint:
name: node20-lint
node-version: lts/iron
- unit-test:
name: node18-chrome
node-version: lts/hydrogen
- e2e-test:
name: e2e-ci
suite: ci
- e2e-mobile
- visual-a11y:
name: visual-a11y-ci
suite: ci
the-nightly: #These jobs do not run on PRs, but against master at night
jobs:
- unit-test:
name: node20-chrome-nightly
node-version: lts/iron
- unit-test:
name: node18-chrome
node-version: lts/hydrogen
- npm-audit:
node-version: lts/hydrogen
- e2e-test:
name: e2e-full-nightly
suite: full
- e2e-mobile
- perf-test
- mem-test
- visual-a11y:
name: visual-a11y-nightly
suite: full
- e2e-couchdb
triggers:
- schedule:
cron: '0 0 * * *'
filters:
branches:
only:
- master