openmct/.circleci/config.yml
John Hill 69b81c00ca
[e2e] Steps to reduce flakiness in test reporting (#7436)
* flakefinder gha

* Update e2e-flakefinder.yml

* driveby

* skip visual

* first attempt at sharding with circle

* Updated config.yml

* fixes

* missing quote

* re-enable old jobs and update to 7x

* max failures

* destructure the npm script

* missing quote

* revert

* uncomment and re-add 7 parallel

* add unit tests

* add p flag

* skip the flaky test
2024-01-30 14:21:52 -08:00

293 lines
10 KiB
YAML

version: 2.1
executors:
pw-focal-development:
docker:
- image: mcr.microsoft.com/playwright:v1.39.0-focal
environment:
NODE_ENV: development # Needed to ensure 'dist' folder created and devDependencies installed
PERCY_POSTINSTALL_BROWSER: "true" # Needed to store the percy browser in cache deps
PERCY_LOGLEVEL: "debug" # Enable DEBUG level logging for Percy (Issue: https://github.com/nasa/openmct/issues/5742)
ubuntu:
machine:
image: ubuntu-2204:current
docker_layer_caching: true
parameters:
BUST_CACHE:
description: "Set this with the CircleCI UI Trigger Workflow button (boolean = true) to bust the cache!"
default: false
type: boolean
commands:
build_and_install:
description: "All steps used to build and install. Will use cache if found"
parameters:
node-version:
type: string
steps:
- checkout
- restore_cache_cmd:
node-version: << parameters.node-version >>
- node/install:
node-version: << parameters.node-version >>
- run: npm install --no-audit --progress=false
restore_cache_cmd:
description: "Custom command for restoring cache with the ability to bust cache. When BUST_CACHE is set to true, jobs will not restore cache"
parameters:
node-version:
type: string
steps:
- when:
condition:
equal: [false, << pipeline.parameters.BUST_CACHE >>]
steps:
- restore_cache:
key: deps--{{ arch }}--{{ .Branch }}--<< parameters.node-version >>--{{ checksum "package.json" }}-{{ checksum ".circleci/config.yml" }}
save_cache_cmd:
description: "Custom command for saving cache."
parameters:
node-version:
type: string
steps:
- save_cache:
key: deps--{{ arch }}--{{ .Branch }}--<< parameters.node-version >>--{{ checksum "package.json" }}-{{ checksum ".circleci/config.yml" }}
paths:
- ~/.npm
- node_modules
generate_and_store_version_and_filesystem_artifacts:
description: "Track important packages and files"
steps:
- run: |
[[ $EUID -ne 0 ]] && (sudo mkdir -p /tmp/artifacts && sudo chmod 777 /tmp/artifacts) || (mkdir -p /tmp/artifacts && chmod 777 /tmp/artifacts)
printenv NODE_ENV >> /tmp/artifacts/NODE_ENV.txt || true
npm -v >> /tmp/artifacts/npm-version.txt
node -v >> /tmp/artifacts/node-version.txt
ls -latR >> /tmp/artifacts/dir.txt
- store_artifacts:
path: /tmp/artifacts/
generate_e2e_code_cov_report:
description: "Generate e2e code coverage artifacts and publish to codecov.io. Needed to that we can ignore the exit code status of the npm run test"
parameters:
suite:
type: string
steps:
- run: npm run cov:e2e:report || true
- run: npm run cov:e2e:<<parameters.suite>>:publish
orbs:
node: circleci/node@5.1.0
browser-tools: circleci/browser-tools@1.3.0
jobs:
npm-audit:
parameters:
node-version:
type: string
executor: pw-focal-development
steps:
- build_and_install:
node-version: <<parameters.node-version>>
- run: npm audit --audit-level=low
- generate_and_store_version_and_filesystem_artifacts
lint:
parameters:
node-version:
type: string
executor: pw-focal-development
steps:
- build_and_install:
node-version: <<parameters.node-version>>
- run: npm run lint
- generate_and_store_version_and_filesystem_artifacts
unit-test:
parameters:
node-version:
type: string
executor: pw-focal-development
steps:
- build_and_install:
node-version: <<parameters.node-version>>
- browser-tools/install-chrome:
replace-existing: false
- run:
command: |
mkdir -p dist/reports/tests/
TESTFILES=$(circleci tests glob "src/**/*Spec.js")
echo "$TESTFILES" | circleci tests run --command="xargs npm run test" --verbose
- run: npm run cov:unit:publish
- save_cache_cmd:
node-version: <<parameters.node-version>>
- store_test_results:
path: dist/reports/tests/
- store_artifacts:
path: coverage
- when:
condition:
equal: [42, 42] # Always generate version artifacts regardless of test failure https://discuss.circleci.com/t/make-custom-command-run-always-with-when-always/38957/2
steps:
- generate_and_store_version_and_filesystem_artifacts
e2e-test:
parameters:
suite: #stable or full
type: string
executor: pw-focal-development
parallelism: 7
steps:
- build_and_install:
node-version: lts/hydrogen
- when: #Only install chrome-beta when running the 'full' suite to save $$$
condition:
equal: ["full", <<parameters.suite>>]
steps:
- run: npx playwright install chrome-beta
- run:
command: |
mkdir test-results
TESTFILES=$(circleci tests glob "e2e/**/*.spec.js")
echo "$TESTFILES" | circleci tests run --command="xargs npm run test:e2e:<<parameters.suite>>" --verbose --split-by=timings
- when:
condition:
equal: [42, 42] # Always run codecov reports regardless of test failure https://discuss.circleci.com/t/make-custom-command-run-always-with-when-always/38957/2
steps:
- generate_e2e_code_cov_report:
suite: <<parameters.suite>>
- store_test_results:
path: test-results/results.xml
- store_artifacts:
path: test-results
- store_artifacts:
path: coverage
- store_artifacts:
path: html-test-results
- when:
condition:
equal: [42, 42] # Always generate version artifacts regardless of test failure https://discuss.circleci.com/t/make-custom-command-run-always-with-when-always/38957/2
steps:
- generate_and_store_version_and_filesystem_artifacts
e2e-couchdb:
executor: ubuntu
steps:
- build_and_install:
node-version: lts/hydrogen
- run: npx playwright@1.39.0 install #Necessary for bare ubuntu machine
- run: |
export $(cat src/plugins/persistence/couch/.env.ci | xargs)
docker-compose -f src/plugins/persistence/couch/couchdb-compose.yaml up --detach
sleep 3
bash src/plugins/persistence/couch/setup-couchdb.sh
- run: sh src/plugins/persistence/couch/replace-localstorage-with-couchdb-indexhtml.sh #Replace LocalStorage Plugin with CouchDB
- run: npm run test:e2e:couchdb
- when:
condition:
equal: [42, 42] # Always run codecov reports regardless of test failure https://discuss.circleci.com/t/make-custom-command-run-always-with-when-always/38957/2
steps:
- generate_e2e_code_cov_report:
suite: full #add to full suite
- store_test_results:
path: test-results/results.xml
- store_artifacts:
path: test-results
- store_artifacts:
path: coverage
- store_artifacts:
path: html-test-results
- when:
condition:
equal: [42, 42] # Always generate version artifacts regardless of test failure https://discuss.circleci.com/t/make-custom-command-run-always-with-when-always/38957/2
steps:
- generate_and_store_version_and_filesystem_artifacts
mem-test:
executor: pw-focal-development
steps:
- build_and_install:
node-version: lts/hydrogen
- run: npm run test:perf:memory
- store_test_results:
path: test-results/results.xml
- store_artifacts:
path: test-results
- store_artifacts:
path: html-test-results
- when:
condition:
equal: [42, 42] # Always run codecov reports regardless of test failure https://discuss.circleci.com/t/make-custom-command-run-always-with-when-always/38957/2
steps:
- generate_and_store_version_and_filesystem_artifacts
perf-test:
executor: pw-focal-development
steps:
- build_and_install:
node-version: lts/hydrogen
- run: npm run test:perf:localhost
- run: npm run test:perf:contract
- store_test_results:
path: test-results/results.xml
- store_artifacts:
path: test-results
- store_artifacts:
path: html-test-results
- when:
condition:
equal: [42, 42] # Always run codecov reports regardless of test failure https://discuss.circleci.com/t/make-custom-command-run-always-with-when-always/38957/2
steps:
- generate_and_store_version_and_filesystem_artifacts
visual-a11y-tests:
parameters:
suite:
type: string # ci or full
executor: pw-focal-development
steps:
- build_and_install:
node-version: lts/hydrogen
- run: npm run test:e2e:visual:<<parameters.suite>>
- store_test_results:
path: test-results/results.xml
- store_artifacts:
path: test-results
- store_artifacts:
path: html-test-results
- when:
condition:
equal: [42, 42] # Always generate version artifacts regardless of test failure https://discuss.circleci.com/t/make-custom-command-run-always-with-when-always/38957/2
steps:
- generate_and_store_version_and_filesystem_artifacts
workflows:
overall-circleci-commit-status: #These jobs run on every commit
jobs:
- lint:
name: node20-lint
node-version: lts/iron
- unit-test:
name: node18-chrome
node-version: lts/hydrogen
- e2e-test:
name: e2e-stable
suite: stable
- visual-a11y-tests:
name: visual-test-ci
suite: ci
the-nightly: #These jobs do not run on PRs, but against master at night
jobs:
- unit-test:
name: node20-chrome-nightly
node-version: lts/iron
- unit-test:
name: node18-chrome
node-version: lts/hydrogen
- npm-audit:
node-version: lts/hydrogen
- e2e-test:
name: e2e-full-nightly
suite: full
- mem-test
- perf-test
- visual-a11y-tests:
name: visual-test-nightly
suite: full
- e2e-couchdb
triggers:
- schedule:
cron: "0 0 * * *"
filters:
branches:
only:
- master