Implement e2e functional smoke testing and visual regression testing with playwright (#4456)

* First example of playwright * Move config * set junit output * fix config and update version * staged changes * new files * update to remote dir * remove remote * smoke * Update smoke.spec.js * Add eslint for playwright * Add dependabot rule for playwright * Add adhoc GHA for playwright * Update tests and fix eslint * move playwright eslint config to e2e dir * Add to circle config * store artifacts * wrong location for storing artifacts * slash? * build before start * increase timeouts in circle * remove duplicate build step * timeout values * add retries * reduce retries * add percy emblem * added percy * Adds GHA for adhoc trigger and baseline visual * Bump and Rev dependabot separte from karma * update e2e label for additional pr logic * Ignore playwright changes for codeql * Update documentation * Add ability to run all tests on pr label * eof * yamlamlaml * issue instead of pull api * archive results and include a link in the PR comment * log context and attempt string concat * concat fix * add success/failure options * add alure reporting * lock playwright image Co-authored-by: unlikelyzero <jchill2@gmail.com>
2024-12-18 20:57:53 +00:00 · 2021-11-29 16:34:47 -08:00 · 2021-11-29 16:34:47 -08:00 · 02c2fb5655
commit 02c2fb5655
parent 0b02b083c3
15 changed files with 389 additions and 7 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -2,7 +2,7 @@ version: 2.1
 executors:
  linux:
    docker:
-      - image: cimg/base:stable
+      - image: mcr.microsoft.com/playwright:focal
 orbs:
  node: circleci/node@4.5.1
  browser-tools: circleci/browser-tools@1.1.3
@ -54,6 +54,7 @@ jobs:
            - ~/.npm
            - ~/.cache
            - node_modules
+            - ~/Library/Caches/ms-playwright
      - when:
          condition:
            equal: [ "", <<parameters.browser>> ] #Only run linting when browsers are not running to save time
@ -67,6 +68,35 @@ jobs:
                path: dist/reports/tests/
            - store_artifacts:
                path: dist/reports/
+  e2e:
+    parameters:
+      node-version:
+        type: string
+      suite:
+        type: string
+    executor: linux
+    environment:
+      NODE_ENV: development # Needed if playwright is in `devDependencies`
+    steps:
+      - checkout
+      - restore_cache:
+          key: deps-{{ .Branch }}--<< parameters.node-version >>--{{ checksum "package.json" }}
+      - node/install:
+          install-npm: false
+          node-version: << parameters.node-version >>
+      - run: npm install
+      - save_cache:
+          key: deps-{{ .Branch }}--<< parameters.node-version >>--{{ checksum "package.json" }}
+          paths:
+            - ~/.npm
+            - ~/.cache
+            - node_modules
+            - ~/Library/Caches/ms-playwright
+      - run: npm run test:e2e:<<parameters.suite>>
+      - store_test_results:
+          path: test-results/results.xml
+      - store_artifacts:
+          path: test-results
 workflows:
  matrix-tests:
    jobs:
@ -86,7 +116,10 @@ workflows:
          name: node14-build-lint
          node-version: lts/fermium
          browser: "" #Skip unit tests
-
+      - e2e:
+          name: e2e-smoke
+          node-version: lts/fermium
+          suite: ci
  nightly:
    jobs:
      - test:
@ -102,12 +135,14 @@ workflows:
          node-version: lts/fermium
          browser: FirefoxHeadless
      - npm-audit
+      - e2e:
+          name: e2e-full
+          node-version: lts/fermium
+          suite: full
    triggers:
      - schedule:
          cron: "0 0 * * *"
          filters:
            branches:
              only:
-                - master      
-
-
+                - master
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@ -13,6 +13,19 @@ updates:
      - dependency-name: "eslint*"
      - dependency-name: "karma*"
      - dependency-name: "jasmine*"
+  
+  - package-ecosystem: "npm"
+    directory: "/"
+    schedule:
+      interval: "daily"  
+    open-pull-requests-limit: 2
+    labels:
+      - "type:maintenance"
+      - "dependencies"
+      - "pr:e2e"
+    allow:
+      - dependency-name: "playwright*"
+      - dependency-name: "percy*"

  - package-ecosystem: "github-actions"
    directory: "/"
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@ -12,6 +12,8 @@ on:
      - '**/*.txt'
      - '**/*.yml'
      - '**/*.yaml'
+      - '**/*.spec.js'
+      - '**/*.config.js'
  schedule:
    - cron: '28 21 * * 3'

--- a/.github/workflows/e2e-pr.yml
+++ b/.github/workflows/e2e-pr.yml
@ -0,0 +1,53 @@
+name: "e2e-pr"
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [ labeled ]
+
+jobs:
+  e2e-full:
+    if: ${{ github.event.label.name == 'pr:e2e' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Trigger Success
+        uses: actions/github-script@v5
+        with:
+          script: |
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: 'Started e2e Run. Follow along: https://github.com/nasa/openmct/actions/runs/' + context.runId
+            })
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+      - run: npm install
+      - run: npm run test:e2e:full
+      - name: Archive test results
+        uses: actions/upload-artifact@v2
+        with:
+          path: test-results
+      - name: Test success
+        if: ${{ success() }}
+        uses: actions/github-script@v5
+        with:
+          script: |
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: 'Success ✅ ! Build artifacts are here: https://github.com/nasa/openmct/actions/runs/' + context.runId
+            })
+      - name: Test failure
+        if: ${{ failure() }}
+        uses: actions/github-script@v5
+        with:
+          script: |
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: 'Failure ❌ ! Build artifacts are here: https://github.com/nasa/openmct/actions/runs/' + context.runId
+            })
--- a/.github/workflows/e2e-visual.yml
+++ b/.github/workflows/e2e-visual.yml
@ -0,0 +1,22 @@
+name: "e2e-visual"
+on:
+  workflow_dispatch:
+    inputs: 
+      version:
+        description: '(Not configured) Which branch do you want to test?' # Limited to branch for now
+        required: false
+        default: 'master'
+  schedule:
+    - cron: '28 21 * * 3'
+
+jobs:
+  e2e-visual:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+      - run: npm install
+      - name: Run the e2e visual tests
+        run: npm run test:e2e:visual
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@ -0,0 +1,21 @@
+name: "e2e"
+on:
+  workflow_dispatch:
+    inputs: 
+      version:
+        description: 'Which branch do you want to test?' # Limited to branch for now
+        required: false
+        default: 'master' 
+jobs:
+  e2e:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.inputs.version }}
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+      - run: npm install
+      - name: Run the e2e tests
+        run: npm run test:e2e:ci
--- a/.gitignore
+++ b/.gitignore
@ -43,6 +43,10 @@ report.*.json
 # Lighthouse reports
 .lighthouseci

+# e2e test artifacts
+test-results
+allure-results
+
 package-lock.json

 #codecov artifacts
--- a/README.md
+++ b/README.md
@ -1,4 +1,4 @@
-# Open MCT [![license](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](http://www.apache.org/licenses/LICENSE-2.0) [![Language grade: JavaScript](https://img.shields.io/lgtm/grade/javascript/g/nasa/openmct.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/nasa/openmct/context:javascript) [![codecov](https://codecov.io/gh/nasa/openmct/branch/master/graph/badge.svg?token=7DQIipp3ej)](https://codecov.io/gh/nasa/openmct)
+# Open MCT [![license](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](http://www.apache.org/licenses/LICENSE-2.0) [![Language grade: JavaScript](https://img.shields.io/lgtm/grade/javascript/g/nasa/openmct.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/nasa/openmct/context:javascript) [![codecov](https://codecov.io/gh/nasa/openmct/branch/master/graph/badge.svg?token=7DQIipp3ej)](https://codecov.io/gh/nasa/openmct) [![This project is using Percy.io for visual regression testing.](https://percy.io/static/images/percy-badge.svg)](https://percy.io/b2e34b17/openmct)

 Open MCT (Open Mission Control Technologies) is a next-generation mission control framework for visualization of data on desktop and mobile devices. It is developed at NASA's Ames Research Center, and is being used by NASA for data analysis of spacecraft missions, as well as planning and operation of experimental rover systems. As a generalizable and open source framework, Open MCT could be used as the basis for building applications for planning, operation, and analysis of any systems producing telemetry data.

--- a/e2e/.eslintrc.js
+++ b/e2e/.eslintrc.js
@ -0,0 +1,4 @@
+/* eslint-disable no-undef */
+module.exports = {
+    "extends": ["plugin:playwright/playwright-test"]
+};
--- a/e2e/playwright-ci.config.js
+++ b/e2e/playwright-ci.config.js
@ -0,0 +1,32 @@
+/* eslint-disable no-undef */
+// playwright.config.js
+// @ts-check
+
+/** @type {import('@playwright/test').PlaywrightTestConfig} */
+const config = {
+    retries: 2,
+    testDir: 'tests',
+    timeout: 90 * 1000,
+    webServer: {
+        command: 'npm run start',
+        port: 8080,
+        timeout: 200 * 1000,
+        reuseExistingServer: !process.env.CI
+    },
+    use: {
+        browserName: "chromium",
+        baseURL: 'http://localhost:8080/',
+        headless: true,
+        ignoreHTTPSErrors: true,
+        screenshot: 'on',
+        trace: 'on',
+        video: 'on'
+    },
+    reporter: [
+        ['list'],
+        ['junit', { outputFile: 'test-results/results.xml' }],
+        ['allure-playwright']
+    ]
+};
+
+module.exports = config;
--- a/e2e/playwright-local.config.js
+++ b/e2e/playwright-local.config.js
@ -0,0 +1,31 @@
+/* eslint-disable no-undef */
+// playwright.config.js
+// @ts-check
+
+/** @type {import('@playwright/test').PlaywrightTestConfig} */
+const config = {
+    retries: 0,
+    testDir: 'tests',
+    timeout: 30 * 1000,
+    webServer: {
+        command: 'npm run start',
+        port: 8080,
+        timeout: 120 * 1000,
+        reuseExistingServer: !process.env.CI
+    },
+    use: {
+        browserName: "chromium",
+        baseURL: 'http://localhost:8080/',
+        headless: false,
+        ignoreHTTPSErrors: true,
+        screenshot: 'on',
+        trace: 'on',
+        video: 'on'
+    },
+    reporter: [
+        ['list'],
+        ['allure-playwright']
+    ]
+};
+
+module.exports = config;
--- a/e2e/playwright-visual.config.js
+++ b/e2e/playwright-visual.config.js
@ -0,0 +1,32 @@
+/* eslint-disable no-undef */
+// playwright.config.js
+// @ts-check
+
+/** @type {import('@playwright/test').PlaywrightTestConfig} */
+const config = {
+    retries: 0,
+    testDir: 'tests',
+    timeout: 90 * 1000,
+    webServer: {
+        command: 'npm run start',
+        port: 8080,
+        timeout: 200 * 1000,
+        reuseExistingServer: !process.env.CI
+    },
+    use: {
+        browserName: "chromium",
+        baseURL: 'http://localhost:8080/',
+        headless: true,
+        ignoreHTTPSErrors: true,
+        screenshot: 'on',
+        trace: 'off',
+        video: 'on'
+    },
+    reporter: [
+        ['list'],
+        ['junit', { outputFile: 'test-results/results.xml' }],
+        ['allure-playwright']
+    ]
+};
+
+module.exports = config;
--- a/e2e/tests/smoke.spec.js
+++ b/e2e/tests/smoke.spec.js
@ -0,0 +1,49 @@
+/*****************************************************************************
+ * Open MCT, Copyright (c) 2014-2021, United States Government
+ * as represented by the Administrator of the National Aeronautics and Space
+ * Administration. All rights reserved.
+ *
+ * Open MCT is licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * Open MCT includes source code licensed under additional open source
+ * licenses. See the Open Source Licenses file (LICENSES.md) included with
+ * this source code distribution or the Licensing information page available
+ * at runtime from the About dialog for additional information.
+ *****************************************************************************/
+
+/*
+This test suite is dedicated to tests which can quickly verify that any openmct installation is
+operable and that any type of testing can proceed.
+
+Ideally, smoke tests should make zero assumptions about how and where they are run. This makes them
+more resilient to change and therefor a better indicator of failure. Smoke tests will also run quickly
+as they cover a very "thin surface" of functionality.
+
+When deciding between authoring new smoke tests or functional tests, ask yourself "would I feel
+comfortable running this test during a live mission?" Avoid creating or deleting Domain Objects.
+Make no assumptions about the order that elements appear in the DOM.
+*/
+
+const { test, expect } = require('@playwright/test');
+
+test('Verify that the create button appears and that the Folder Domain Object is available for selection', async ({ page }) => {
+
+    //Go to baseURL
+    await page.goto('/', { waitUntil: 'networkidle' });
+
+    //Click the Create button
+    await page.click('button:has-text("Create")');
+
+    // Verify that Create Folder appears in the dropdown
+    const locator = page.locator(':nth-match(:text("Folder"), 2)');
+    await expect(locator).toBeEnabled();
+});
--- a/e2e/tests/visual/default.spec.js
+++ b/e2e/tests/visual/default.spec.js
@ -0,0 +1,73 @@
+/*****************************************************************************
+ * Open MCT, Copyright (c) 2014-2021, United States Government
+ * as represented by the Administrator of the National Aeronautics and Space
+ * Administration. All rights reserved.
+ *
+ * Open MCT is licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * Open MCT includes source code licensed under additional open source
+ * licenses. See the Open Source Licenses file (LICENSES.md) included with
+ * this source code distribution or the Licensing information page available
+ * at runtime from the About dialog for additional information.
+ *****************************************************************************/
+
+/*
+Collection of Visual Tests set to run in a default context. These should only use functional
+expect statements to verify assumptions about the state in a test and not for functional
+verification of correctness.
+Note: Larger testsuite sizes are OK due to the setup time associated with these tests. Visual
+tests are not supposed to "fail" on assertions.
+*/
+
+const { test, expect } = require('@playwright/test');
+const percySnapshot = require('@percy/playwright');
+const path = require('path');
+const sinon = require('sinon');
+
+const VISUAL_GRACE_PERIOD = 5 * 1000; //Lets the application "simmer" before the snapshot is taken
+
+// Snippet from https://github.com/microsoft/playwright/issues/6347#issuecomment-965887758
+// Will replace with cy.clock() equivalent
+test.beforeEach(async ({ context }) => {
+    await context.addInitScript({
+        // eslint-disable-next-line no-undef
+        path: path.join(__dirname, '../../..', './node_modules/sinon/pkg/sinon.js')
+    });
+    await context.addInitScript(() => {
+        window.__clock = sinon.useFakeTimers(); //Set browser clock to UNIX Epoch
+    });
+});
+
+test('Visual - Root and About', async ({ page }) => {
+    // Go to baseURL
+    await page.goto('/', { waitUntil: 'networkidle' });
+
+    // Verify that Create button is actionable
+    const createButtonLocator = page.locator('button:has-text("Create")');
+    await expect(createButtonLocator).toBeEnabled();
+
+    // Take a snapshot of the Dashboard
+    await page.waitForTimeout(VISUAL_GRACE_PERIOD);
+    await percySnapshot(page, 'Root');
+
+    // Click About button
+    await page.click('.l-shell__app-logo');
+
+    // Modify the Build information in 'about' to be consistent run-over-run
+    const versionInformationLocator = page.locator('ul.t-info.l-info.s-info');
+    await expect(versionInformationLocator).toBeEnabled();
+    await versionInformationLocator.evaluate(node => node.innerHTML = '<li>Version: visual-snapshot</li> <li>Build Date: Mon Nov 15 2021 08:07:51 GMT-0800 (Pacific Standard Time)</li> <li>Revision: 93049cdbc6c047697ca204893db9603b864b8c9f</li> <li>Branch: master</li>');
+
+    // Take a snapshot of the About modal
+    await page.waitForTimeout(VISUAL_GRACE_PERIOD);
+    await percySnapshot(page, 'About');
+});
--- a/package.json
+++ b/package.json
@ -4,6 +4,10 @@
  "description": "The Open MCT core platform",
  "devDependencies": {
    "@braintree/sanitize-url": "^5.0.2",
+    "@percy/cli": "^1.0.0-beta.70",
+    "@percy/playwright": "^1.0.1",
+    "@playwright/test": "^1.16.3",
+    "allure-playwright": "^2.0.0-beta.14",
    "angular": ">=1.8.0",
    "angular-route": "1.4.14",
    "babel-eslint": "10.0.3",
@ -16,6 +20,7 @@
    "d3-scale": "1.0.x",
    "d3-selection": "1.3.x",
    "eslint": "7.0.0",
+    "eslint-plugin-playwright": "0.6.0",
    "eslint-plugin-vue": "^7.5.0",
    "eslint-plugin-you-dont-need-lodash-underscore": "^6.10.0",
    "eventemitter3": "^1.2.0",
@ -55,11 +60,13 @@
    "node-bourbon": "^4.2.3",
    "node-sass": "^4.14.1",
    "painterro": "^1.2.56",
+    "playwright": "^1.16.3",
    "plotly.js-basic-dist": "^2.5.0",
    "plotly.js-gl2d-dist": "^2.5.0",
    "printj": "^1.2.1",
    "raw-loader": "^0.5.1",
    "request": "^2.69.0",
+    "sinon": "^12.0.1",
    "split": "^1.0.0",
    "style-loader": "^1.0.1",
    "uuid": "^3.3.3",
@ -87,6 +94,10 @@
    "test:debug": "cross-env NODE_ENV=debug karma start --no-single-run",
    "test:coverage": "cross-env NODE_OPTIONS=\"--max_old_space_size=4096\" COVERAGE=true karma start --single-run",
    "test:coverage:firefox": "cross-env NODE_OPTIONS=\"--max_old_space_size=4096\" karma start --single-run --browsers=FirefoxHeadless",
+    "test:e2e:ci": "npx playwright test --config=e2e/playwright-ci.config.js smoke",
+    "test:e2e:local": "npx playwright test --config=e2e/playwright-local.config.js",
+    "test:e2e:visual": "percy exec -- npx playwright test --config=e2e/playwright-visual.config.js default",
+    "test:e2e:full": "npx playwright test --config=e2e/playwright-ci.config.js",
    "test:watch": "cross-env NODE_OPTIONS=\"--max_old_space_size=4096\" karma start --no-single-run",
    "verify": "concurrently 'npm:test' 'npm:lint'",
    "jsdoc": "jsdoc -c jsdoc.json -R API.md -r -d dist/docs/api",
@ -99,7 +110,7 @@
    "url": "https://github.com/nasa/openmct.git"
  },
  "engines": {
-    "node": ">=10.10.2 <16.0.0"
+    "node": ">=10.12.2 <16.0.0"
  },
  "author": "",
  "license": "Apache-2.0",