move performance tests to GHA (#7412)

* move performance tests to GHA * no need for chrome beta * Add baseline imagery test * skip flaky app * lint
2024-12-18 20:57:53 +00:00 · 2024-01-24 21:26:48 -08:00 · 2024-01-24 21:26:48 -08:00 · 450cab428f
commit 450cab428f
parent 0340fe18fa
4 changed files with 217 additions and 66 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -5,20 +5,20 @@ executors:
      - image: mcr.microsoft.com/playwright:v1.39.0-focal
    environment:
      NODE_ENV: development # Needed to ensure 'dist' folder created and devDependencies installed
-      PERCY_POSTINSTALL_BROWSER: 'true' # Needed to store the percy browser in cache deps
-      PERCY_LOGLEVEL: 'debug' # Enable DEBUG level logging for Percy (Issue: https://github.com/nasa/openmct/issues/5742)
+      PERCY_POSTINSTALL_BROWSER: "true" # Needed to store the percy browser in cache deps
+      PERCY_LOGLEVEL: "debug" # Enable DEBUG level logging for Percy (Issue: https://github.com/nasa/openmct/issues/5742)
  ubuntu:
    machine:
      image: ubuntu-2204:current
      docker_layer_caching: true
 parameters:
  BUST_CACHE:
-    description: 'Set this with the CircleCI UI Trigger Workflow button (boolean = true) to bust the cache!'
+    description: "Set this with the CircleCI UI Trigger Workflow button (boolean = true) to bust the cache!"
    default: false
    type: boolean
 commands:
  build_and_install:
-    description: 'All steps used to build and install. Will use cache if found'
+    description: "All steps used to build and install. Will use cache if found"
    parameters:
      node-version:
        type: string
@ -30,7 +30,7 @@ commands:
          node-version: << parameters.node-version >>
      - run: npm install --no-audit --progress=false
  restore_cache_cmd:
-    description: 'Custom command for restoring cache with the ability to bust cache. When BUST_CACHE is set to true, jobs will not restore cache'
+    description: "Custom command for restoring cache with the ability to bust cache. When BUST_CACHE is set to true, jobs will not restore cache"
    parameters:
      node-version:
        type: string
@ -42,7 +42,7 @@ commands:
            - restore_cache:
                key: deps--{{ arch }}--{{ .Branch }}--<< parameters.node-version >>--{{ checksum "package.json" }}-{{ checksum ".circleci/config.yml" }}
  save_cache_cmd:
-    description: 'Custom command for saving cache.'
+    description: "Custom command for saving cache."
    parameters:
      node-version:
        type: string
@ -53,7 +53,7 @@ commands:
            - ~/.npm
            - node_modules
  generate_and_store_version_and_filesystem_artifacts:
-    description: 'Track important packages and files'
+    description: "Track important packages and files"
    steps:
      - run: |
          [[ $EUID -ne 0 ]] && (sudo mkdir -p /tmp/artifacts && sudo chmod 777 /tmp/artifacts) || (mkdir -p /tmp/artifacts && chmod 777 /tmp/artifacts)
@ -64,7 +64,7 @@ commands:
      - store_artifacts:
          path: /tmp/artifacts/
  generate_e2e_code_cov_report:
-    description: 'Generate e2e code coverage artifacts and publish to codecov.io. Needed to that we can ignore the exit code status of the npm run test'
+    description: "Generate e2e code coverage artifacts and publish to codecov.io. Needed to that we can ignore the exit code status of the npm run test"
    parameters:
      suite:
        type: string
@ -129,7 +129,7 @@ jobs:
          node-version: lts/hydrogen
      - when: #Only install chrome-beta when running the 'full' suite to save $$$
          condition:
-            equal: ['full', <<parameters.suite>>]
+            equal: ["full", <<parameters.suite>>]
          steps:
            - run: npx playwright install chrome-beta
      - run: SHARD="$((${CIRCLE_NODE_INDEX}+1))"; npm run test:e2e:<<parameters.suite>> -- --shard=${SHARD}/${CIRCLE_NODE_TOTAL}
@ -251,8 +251,6 @@ workflows:
      - e2e-test:
          name: e2e-stable
          suite: stable
-      - mem-test
-      - perf-test
      - visual-a11y-tests:
          name: visual-test-ci
          suite: ci
@ -278,7 +276,7 @@ workflows:
      - e2e-couchdb
    triggers:
      - schedule:
-          cron: '0 0 * * *'
+          cron: "0 0 * * *"
          filters:
            branches:
              only:
--- a/.github/workflows/e2e-perf.yml
+++ b/.github/workflows/e2e-perf.yml
@ -0,0 +1,58 @@
+name: 'e2e-perf'
+on:
+  push:
+    branches: master
+  workflow_dispatch:
+  pull_request:
+    types:
+      - labeled
+      - opened
+  schedule:
+    - cron: '0 0 * * *'
+jobs:
+  e2e-full:
+    if: contains(github.event.pull_request.labels.*.name, 'pr:e2e:perf') || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 'lts/hydrogen'
+
+      - name: Cache NPM dependencies
+        uses: actions/cache@v3
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+
+      - run: npx playwright@1.39.0 install
+      - run: npm install --cache ~/.npm --no-audit --progress=false
+      - run: npm run test:perf:localhost
+      - run: npm run test:perf:contract
+      - run: npm run test:perf:memory
+      - name: Archive test results
+        if: success() || failure()
+        uses: actions/upload-artifact@v3
+        with:
+          path: test-results
+
+      - name: Remove pr:e2e:perf label (if present)
+        if: always()
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const { owner, repo, number } = context.issue;
+            const labelToRemove = 'pr:e2e:perf';
+            try {
+              await github.rest.issues.removeLabel({
+                owner,
+                repo,
+                issue_number: number,
+                name: labelToRemove
+              });
+            } catch (error) {
+              core.warning(`Failed to remove ' + labelToRemove + ' label: ${error.message}`);
+            }
--- a/e2e/tests/functional/plugins/plot/overlayPlot.e2e.spec.js
+++ b/e2e/tests/functional/plugins/plot/overlayPlot.e2e.spec.js
@ -63,78 +63,80 @@ test.describe('Overlay Plot', () => {
    await expect(seriesColorSwatch).toHaveCSS('background-color', 'rgb(255, 166, 61)');
  });

-  test('Limit lines persist when series is moved to another Y Axis and on refresh', async ({
-    page
-  }) => {
-    test.info().annotations.push({
-      type: 'issue',
-      description: 'https://github.com/nasa/openmct/issues/6338'
-    });
-    // Create an Overlay Plot with a default SWG
-    const overlayPlot = await createDomainObjectWithDefaults(page, {
-      type: 'Overlay Plot'
-    });
+  //skipping due to https://github.com/nasa/openmct/issues/7405
+  test.fixme(
+    'Limit lines persist when series is moved to another Y Axis and on refresh',
+    async ({ page }) => {
+      test.info().annotations.push({
+        type: 'issue',
+        description: 'https://github.com/nasa/openmct/issues/6338'
+      });
+      // Create an Overlay Plot with a default SWG
+      const overlayPlot = await createDomainObjectWithDefaults(page, {
+        type: 'Overlay Plot'
+      });

-    const swgA = await createDomainObjectWithDefaults(page, {
-      type: 'Sine Wave Generator',
-      parent: overlayPlot.uuid
-    });
+      const swgA = await createDomainObjectWithDefaults(page, {
+        type: 'Sine Wave Generator',
+        parent: overlayPlot.uuid
+      });

-    await page.goto(overlayPlot.url);
+      await page.goto(overlayPlot.url);

-    // Assert that no limit lines are shown by default
-    await page.waitForSelector('.js-limit-area', { state: 'attached' });
-    expect(await page.locator('.c-plot-limit-line').count()).toBe(0);
+      // Assert that no limit lines are shown by default
+      await page.waitForSelector('.js-limit-area', { state: 'attached' });
+      expect(await page.locator('.c-plot-limit-line').count()).toBe(0);

-    // Enter edit mode
-    await page.getByLabel('Edit Object').click();
+      // Enter edit mode
+      await page.getByLabel('Edit Object').click();

-    // Expand the "Sine Wave Generator" plot series options and enable limit lines
-    await page.getByRole('tab', { name: 'Config' }).click();
-    await page
-      .getByRole('list', { name: 'Plot Series Properties' })
-      .locator('span')
-      .first()
-      .click();
-    await page
-      .getByRole('list', { name: 'Plot Series Properties' })
-      .locator('[title="Display limit lines"]~div input')
-      .check();
+      // Expand the "Sine Wave Generator" plot series options and enable limit lines
+      await page.getByRole('tab', { name: 'Config' }).click();
+      await page
+        .getByRole('list', { name: 'Plot Series Properties' })
+        .locator('span')
+        .first()
+        .click();
+      await page
+        .getByRole('list', { name: 'Plot Series Properties' })
+        .locator('[title="Display limit lines"]~div input')
+        .check();

-    await assertLimitLinesExistAndAreVisible(page);
+      await assertLimitLinesExistAndAreVisible(page);

-    // Save (exit edit mode)
-    await page.locator('button[title="Save"]').click();
-    await page.getByRole('listitem', { name: 'Save and Finish Editing' }).click();
+      // Save (exit edit mode)
+      await page.locator('button[title="Save"]').click();
+      await page.getByRole('listitem', { name: 'Save and Finish Editing' }).click();

-    await assertLimitLinesExistAndAreVisible(page);
+      await assertLimitLinesExistAndAreVisible(page);

-    await page.reload();
+      await page.reload();

-    await assertLimitLinesExistAndAreVisible(page);
+      await assertLimitLinesExistAndAreVisible(page);

-    // Enter edit mode
-    await page.getByLabel('Edit Object').click();
+      // Enter edit mode
+      await page.getByLabel('Edit Object').click();

-    await page.getByRole('tab', { name: 'Elements' }).click();
+      await page.getByRole('tab', { name: 'Elements' }).click();

-    // Drag Sine Wave Generator series from Y Axis 1 into Y Axis 2
-    await page
-      .locator(`#inspector-elements-tree >> text=${swgA.name}`)
-      .dragTo(page.locator('[aria-label="Element Item Group Y Axis 2"]'));
+      // Drag Sine Wave Generator series from Y Axis 1 into Y Axis 2
+      await page
+        .locator(`#inspector-elements-tree >> text=${swgA.name}`)
+        .dragTo(page.locator('[aria-label="Element Item Group Y Axis 2"]'));

-    await assertLimitLinesExistAndAreVisible(page);
+      await assertLimitLinesExistAndAreVisible(page);

-    // Save (exit edit mode)
-    await page.locator('button[title="Save"]').click();
-    await page.getByRole('listitem', { name: 'Save and Finish Editing' }).click();
+      // Save (exit edit mode)
+      await page.locator('button[title="Save"]').click();
+      await page.getByRole('listitem', { name: 'Save and Finish Editing' }).click();

-    await assertLimitLinesExistAndAreVisible(page);
+      await assertLimitLinesExistAndAreVisible(page);

-    await page.reload();
+      await page.reload();

-    await assertLimitLinesExistAndAreVisible(page);
-  });
+      await assertLimitLinesExistAndAreVisible(page);
+    }
+  );

  test('The elements pool supports dragging series into multiple y-axis buckets', async ({
    page
--- a/e2e/tests/visual-a11y/imagery.visual.spec.js
+++ b/e2e/tests/visual-a11y/imagery.visual.spec.js
@ -0,0 +1,93 @@
+/*****************************************************************************
+ * Open MCT, Copyright (c) 2014-2024, United States Government
+ * as represented by the Administrator of the National Aeronautics and Space
+ * Administration. All rights reserved.
+ *
+ * Open MCT is licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * Open MCT includes source code licensed under additional open source
+ * licenses. See the Open Source Licenses file (LICENSES.md) included with
+ * this source code distribution or the Licensing information page available
+ * at runtime from the About dialog for additional information.
+ *****************************************************************************/
+
+import percySnapshot from '@percy/playwright';
+
+import { createDomainObjectWithDefaults, setRealTimeMode } from '../../appActions.js';
+import { VISUAL_URL } from '../../constants.js';
+import { expect, test } from '../../pluginFixtures.js';
+
+test.describe('Visual - Example Imagery', () => {
+  let exampleImagery;
+  let parentLayout;
+
+  test.beforeEach(async ({ page }) => {
+    await page.goto(VISUAL_URL, { waitUntil: 'domcontentloaded' });
+
+    parentLayout = await createDomainObjectWithDefaults(page, {
+      type: 'Display Layout',
+      name: 'Parent Layout'
+    });
+
+    exampleImagery = await createDomainObjectWithDefaults(page, {
+      type: 'Example Imagery',
+      name: 'Example Imagery Test',
+      parent: parentLayout.uuid
+    });
+
+    // Modify Example Imagery to create a really stable Example Imagery
+    await page.goto(exampleImagery.url, { waitUntil: 'domcontentloaded' });
+    await page.getByRole('button', { name: 'More actions' }).click();
+    await page.getByRole('menuitem', { name: 'Edit Properties...' }).click();
+    await page
+      .locator('#imageLocation-textarea')
+      .fill(
+        'https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg,https://www.nasa.gov/wp-content/uploads/static/history/alsj/a16/AS16-117-18731.jpg'
+      );
+    await page.getByRole('button', { name: 'Save' }).click();
+    await page.reload({ waitUntil: 'domcontentloaded' });
+    await page.getByTitle('Collapse Browse Pane').click();
+    await page.getByTitle('Collapse Inspect Pane').click();
+  });
+
+  test('Example Imagery in Fixed Time', async ({ page, theme }) => {
+    await page.goto(exampleImagery.url, { waitUntil: 'domcontentloaded' });
+
+    await expect(page.getByLabel('Image Wrapper')).toBeVisible();
+
+    await percySnapshot(page, `Example Imagery in Fixed Time (theme: ${theme})`);
+
+    await page.getByLabel('Image Wrapper').hover();
+
+    await percySnapshot(page, `Example Imagery Hover in Fixed Time (theme: ${theme})`);
+  });
+
+  test('Example Imagery in Real Time', async ({ page, theme }) => {
+    await page.goto(exampleImagery.url, { waitUntil: 'domcontentloaded' });
+
+    await setRealTimeMode(page, true);
+    //Temporary to close the dialog
+    await page.getByLabel('Submit time offsets').click();
+
+    await expect(page.getByLabel('Image Wrapper')).toBeVisible();
+
+    await percySnapshot(page, `Example Imagery in Real Time (theme: ${theme})`);
+  });
+
+  test('Example Imagery in Display Layout', async ({ page, theme }) => {
+    await page.goto(parentLayout.url, { waitUntil: 'domcontentloaded' });
+
+    await expect(page.getByLabel('Image Wrapper')).toBeVisible();
+
+    await percySnapshot(page, `Example Imagery in Display Layout (theme: ${theme})`);
+  });
+});