First attempt at DeploySentinel and fix couchdb notebook tests (#6398)

* First attempt

* Remove commented pattern

* Add deploysentinel to github action

* drive by

* Stablization

* remove only

* entries now selected on creation

* select previous entry on deletion

* add deletion test

* wip

* fix adding focus selection

* remove previous entry selection logic

* null check for event

* address review comments

* address review comments

* refactor tests a bit

* typo

* Add some determinism to avoid console errors

* refactor and use methods

* stabilize

* remove debug

* remove only

* combine clean commands

* comments

* change to expects

* test: await toBeHidden() assertion

* test: use `myItemsFolderName` instead of 'My Items'

---------

Co-authored-by: Scott Bell <scott@traclabs.com>
Co-authored-by: Jesse Mazzella <jesse.d.mazzella@nasa.gov>

.github/workflows/e2e-couchdb.yml

@@ -26,7 +26,10 @@ jobs:
       - run: npx playwright@1.29.0 install
       - run: npm install
       - run: sh src/plugins/persistence/couch/replace-localstorage-with-couchdb-indexhtml.sh
-      - run: npm run test:e2e:couchdb
+      - name: Run CouchDB Tests and publish to deploysentinel
+        env: 
+          DEPLOYSENTINEL_API_KEY: ${{ secrets.DEPLOYSENTINEL_API_KEY }} 
+        run: npm run test:e2e:couchdb
       - run: ls -latr
       - name: Archive test results
         uses: actions/upload-artifact@v3

e2e/appActions.js

@@ -140,6 +140,7 @@ async function createNotification(page, createNotificationOptions) {
 }
 
 /**
+ * Expand an item in the tree by a given object name.
  * @param {import('@playwright/test').Page} page
  * @param {string} name
  */
@@ -272,6 +273,7 @@ async function getFocusedObjectUuid(page) {
  * @returns {Promise<string>} the url of the object
  */
 async function getHashUrlToDomainObject(page, uuid) {
+    await page.waitForLoadState('load'); //Add some determinism
     const hashUrl = await page.evaluate(async (objectUuid) => {
         const path = await window.openmct.objects.getOriginalPath(objectUuid);
         let url = './#/browse/' + [...path].reverse()

e2e/helper/notebookUtils.js

@@ -28,7 +28,7 @@ const NOTEBOOK_DROP_AREA = '.c-notebook__drag-area';
  * @param {import('@playwright/test').Page} page
  */
 async function enterTextEntry(page, text) {
-    // Click .c-notebook__drag-area
+    // Click the 'Add Notebook Entry' area
     await page.locator(NOTEBOOK_DROP_AREA).click();
 
     // enter text
@@ -58,6 +58,7 @@ async function dragAndDropEmbed(page, notebookObject) {
  * @param {import('@playwright/test').Page} page
  */
 async function commitEntry(page) {
+    //Click the Commit Entry button
     await page.locator('.c-ne__save-button > button').click();
 }
 

e2e/playwright-ci.config.js

@@ -74,7 +74,8 @@ const config = {
             outputFolder: '../html-test-results' //Must be in different location due to https://github.com/microsoft/playwright/issues/12840
         }],
         ['junit', { outputFile: '../test-results/results.xml' }],
-        ['github']
+        ['github'],
+        ['@deploysentinel/playwright']
     ]
 };
 

e2e/tests/functional/forms.e2e.spec.js

@@ -166,12 +166,13 @@ test.describe('Persistence operations @couchdb', () => {
             timeout: 1000
         }).toEqual(1);
     });
-    test('Can create an object after a conflict error @couchdb @2p', async ({ page }) => {
+    test('Can create an object after a conflict error @couchdb @2p', async ({ page, openmctConfig }) => {
         test.info().annotations.push({
             type: 'issue',
             description: 'https://github.com/nasa/openmct/issues/5982'
         });
-
+        const { myItemsFolderName } = openmctConfig;
+        // Instantiate a second page/tab
         const page2 = await page.context().newPage();
 
         // Both pages: Go to baseURL
@@ -180,6 +181,10 @@ test.describe('Persistence operations @couchdb', () => {
             page2.goto('./', { waitUntil: 'networkidle' })
         ]);
 
+        //Slow down the test a bit
+        await expect(page.getByRole('treeitem', { name: `  ${myItemsFolderName}` })).toBeVisible();
+        await expect(page2.getByRole('treeitem', { name: `  ${myItemsFolderName}` })).toBeVisible();
+
         // Both pages: Click the Create button
         await Promise.all([
             page.click('button:has-text("Create")'),

e2e/tests/functional/plugins/notebook/notebookWithCouchDB.e2e.spec.js

@@ -26,46 +26,49 @@ This test suite is dedicated to tests which verify the basic operations surround
 
 const { test, expect } = require('../../../../pluginFixtures');
 const { createDomainObjectWithDefaults } = require('../../../../appActions');
+const nbUtils = require('../../../../helper/notebookUtils');
 
 test.describe('Notebook Tests with CouchDB @couchdb', () => {
     let testNotebook;
+
     test.beforeEach(async ({ page }) => {
         //Navigate to baseURL
         await page.goto('./', { waitUntil: 'networkidle' });
 
         // Create Notebook
-        testNotebook = await createDomainObjectWithDefaults(page, {
-            type: 'Notebook',
-            name: "TestNotebook"
-        });
+        testNotebook = await createDomainObjectWithDefaults(page, {type: 'Notebook' });
+        await page.goto(testNotebook.url, { waitUntil: 'networkidle'});
     });
 
     test('Inspect Notebook Entry Network Requests', async ({ page }) => {
+        //Ensure we're on the annotations Tab in the inspector
         await page.getByText('Annotations').click();
         // Expand sidebar
         await page.locator('.c-notebook__toggle-nav-button').click();
 
         // Collect all request events to count and assert after notebook action
-        let addingNotebookElementsRequests = [];
-        page.on('request', (request) => addingNotebookElementsRequests.push(request));
+        let notebookElementsRequests = [];
+        page.on('request', (request) => notebookElementsRequests.push(request));
 
+        //Clicking Add Page generates
         let [notebookUrlRequest, allDocsRequest] = await Promise.all([
             // Waits for the next request with the specified url
             page.waitForRequest(`**/openmct/${testNotebook.uuid}`),
             page.waitForRequest('**/openmct/_all_docs?include_docs=true'),
             // Triggers the request
-            page.click('[aria-label="Add Page"]'),
-            // Ensures that there are no other network requests
-            page.waitForLoadState('networkidle')
+            page.click('[aria-label="Add Page"]')
         ]);
+        // Ensures that there are no other network requests
+        await page.waitForLoadState('networkidle');
+
         // Assert that only two requests are made
         // Network Requests are:
         // 1) The actual POST to create the page
         // 2) The shared worker event from 👆 request
-        expect(addingNotebookElementsRequests.length).toBe(2);
+        expect(notebookElementsRequests.length).toBe(2);
 
         // Assert on request object
-        expect(notebookUrlRequest.postDataJSON().metadata.name).toBe('TestNotebook');
+        expect(notebookUrlRequest.postDataJSON().metadata.name).toBe(testNotebook.name);
         expect(notebookUrlRequest.postDataJSON().model.persisted).toBeGreaterThanOrEqual(notebookUrlRequest.postDataJSON().model.modified);
         expect(allDocsRequest.postDataJSON().keys).toContain(testNotebook.uuid);
 
@@ -73,13 +76,10 @@ test.describe('Notebook Tests with CouchDB @couchdb', () => {
         // Network Requests are:
         // 1) The actual POST to create the entry
         // 2) The shared worker event from 👆 POST request
-        addingNotebookElementsRequests = [];
-        await page.locator('text=To start a new entry, click here or drag and drop any object').click();
-        await page.locator('[aria-label="Notebook Entry Input"]').click();
-        await page.locator('[aria-label="Notebook Entry Input"]').fill(`First Entry`);
-        await page.locator('[aria-label="Notebook Entry Input"]').press('Enter');
+        notebookElementsRequests = [];
+        await nbUtils.enterTextEntry(page, 'First Entry');
         await page.waitForLoadState('networkidle');
-        expect(addingNotebookElementsRequests.length).toBeLessThanOrEqual(2);
+        expect(notebookElementsRequests.length).toBeLessThanOrEqual(2);
 
         // Add some tags
         // Network Requests are for each tag creation are:
@@ -95,32 +95,17 @@ test.describe('Notebook Tests with CouchDB @couchdb', () => {
         // 10) Entry is timestamped
         // 11) The shared worker event from 👆 POST request
 
-        addingNotebookElementsRequests = [];
-        await page.hover(`button:has-text("Add Tag")`);
-        await page.locator(`button:has-text("Add Tag")`).click();
-        await page.locator('[placeholder="Type to select tag"]').click();
-        await page.locator('[aria-label="Autocomplete Options"] >> text=Driving').click();
-        await page.waitForSelector('[aria-label="Tag"]:has-text("Driving")');
-        page.waitForLoadState('networkidle');
-        expect(filterNonFetchRequests(addingNotebookElementsRequests).length).toBeLessThanOrEqual(11);
+        notebookElementsRequests = [];
+        await addTagAndAwaitNetwork(page, 'Driving');
+        expect(filterNonFetchRequests(notebookElementsRequests).length).toBeLessThanOrEqual(11);
 
-        addingNotebookElementsRequests = [];
-        await page.hover(`button:has-text("Add Tag")`);
-        await page.locator(`button:has-text("Add Tag")`).click();
-        await page.locator('[placeholder="Type to select tag"]').click();
-        await page.locator('[aria-label="Autocomplete Options"] >> text=Drilling').click();
-        await page.waitForSelector('[aria-label="Tag"]:has-text("Drilling")');
-        page.waitForLoadState('networkidle');
-        expect(filterNonFetchRequests(addingNotebookElementsRequests).length).toBeLessThanOrEqual(11);
+        notebookElementsRequests = [];
+        await addTagAndAwaitNetwork(page, 'Drilling');
+        expect(filterNonFetchRequests(notebookElementsRequests).length).toBeLessThanOrEqual(11);
 
-        addingNotebookElementsRequests = [];
-        await page.hover(`button:has-text("Add Tag")`);
-        await page.locator(`button:has-text("Add Tag")`).click();
-        await page.locator('[placeholder="Type to select tag"]').click();
-        await page.locator('[aria-label="Autocomplete Options"] >> text=Science').click();
-        await page.waitForSelector('[aria-label="Tag"]:has-text("Science")');
-        page.waitForLoadState('networkidle');
-        expect(filterNonFetchRequests(addingNotebookElementsRequests).length).toBeLessThanOrEqual(11);
+        notebookElementsRequests = [];
+        await addTagAndAwaitNetwork(page, 'Science');
+        expect(filterNonFetchRequests(notebookElementsRequests).length).toBeLessThanOrEqual(11);
 
         // Delete all the tags
         // Network requests are:
@@ -129,58 +114,25 @@ test.describe('Notebook Tests with CouchDB @couchdb', () => {
         // 3) Timestamp update on entry
         // 4) The shared worker event from 👆 POST request
         // This happens for 3 tags so 12 requests
-        addingNotebookElementsRequests = [];
-        await page.hover('[aria-label="Tag"]:has-text("Driving")');
-        await page.locator('[aria-label="Remove tag Driving"]').click();
-        await page.waitForSelector('[aria-label="Tag"]:has-text("Driving")', {state: 'hidden'});
-        await page.hover('[aria-label="Tag"]:has-text("Drilling")');
-        await page.locator('[aria-label="Remove tag Drilling"]').click();
-        await page.waitForSelector('[aria-label="Tag"]:has-text("Drilling")', {state: 'hidden'});
-        page.hover('[aria-label="Tag"]:has-text("Science")');
-        await page.locator('[aria-label="Remove tag Science"]').click();
-        await page.waitForSelector('[aria-label="Tag"]:has-text("Science")', {state: 'hidden'});
-        page.waitForLoadState('networkidle');
-        expect(filterNonFetchRequests(addingNotebookElementsRequests).length).toBeLessThanOrEqual(12);
+        notebookElementsRequests = [];
+        await removeTagAndAwaitNetwork(page, 'Driving');
+        await removeTagAndAwaitNetwork(page, 'Drilling');
+        await removeTagAndAwaitNetwork(page, 'Science');
+        expect(filterNonFetchRequests(notebookElementsRequests).length).toBeLessThanOrEqual(12);
 
         // Add two more pages
         await page.click('[aria-label="Add Page"]');
         await page.click('[aria-label="Add Page"]');
 
         // Add three entries
-        await page.locator('text=To start a new entry, click here or drag and drop any object').click();
-        await page.locator('[aria-label="Notebook Entry Input"]').click();
-        await page.locator('[aria-label="Notebook Entry Input"]').fill(`First Entry`);
-        await page.locator('[aria-label="Notebook Entry Input"]').press('Enter');
-
-        await page.locator('text=To start a new entry, click here or drag and drop any object').click();
-        await page.locator('[aria-label="Notebook Entry Input"] >> nth=1').click();
-        await page.locator('[aria-label="Notebook Entry Input"] >> nth=1').fill(`Second Entry`);
-        await page.locator('[aria-label="Notebook Entry Input"] >> nth=1').press('Enter');
-
-        await page.locator('text=To start a new entry, click here or drag and drop any object').click();
-        await page.locator('[aria-label="Notebook Entry Input"] >> nth=2').click();
-        await page.locator('[aria-label="Notebook Entry Input"] >> nth=2').fill(`Third Entry`);
-        await page.locator('[aria-label="Notebook Entry Input"] >> nth=2').press('Enter');
+        await nbUtils.enterTextEntry(page, 'First Entry');
+        await nbUtils.enterTextEntry(page, 'Second Entry');
+        await nbUtils.enterTextEntry(page, 'Third Entry');
 
         // Add three tags
-        await page.hover(`button:has-text("Add Tag")`);
-        await page.locator(`button:has-text("Add Tag")`).click();
-        await page.locator('[placeholder="Type to select tag"]').click();
-        await page.locator('[aria-label="Autocomplete Options"] >> text=Science').click();
-        await page.waitForSelector('[aria-label="Tag"]:has-text("Science")');
-
-        await page.hover(`button:has-text("Add Tag")`);
-        await page.locator(`button:has-text("Add Tag")`).click();
-        await page.locator('[placeholder="Type to select tag"]').click();
-        await page.locator('[aria-label="Autocomplete Options"] >> text=Drilling').click();
-        await page.waitForSelector('[aria-label="Tag"]:has-text("Drilling")');
-
-        await page.hover(`button:has-text("Add Tag")`);
-        await page.locator(`button:has-text("Add Tag")`).click();
-        await page.locator('[placeholder="Type to select tag"]').click();
-        await page.locator('[aria-label="Autocomplete Options"] >> text=Driving').click();
-        await page.waitForSelector('[aria-label="Tag"]:has-text("Driving")');
-        page.waitForLoadState('networkidle');
+        await addTagAndAwaitNetwork(page, 'Science');
+        await addTagAndAwaitNetwork(page, 'Drilling');
+        await addTagAndAwaitNetwork(page, 'Driving');
 
         // Add a fourth entry
         // Network requests are:
@@ -188,14 +140,11 @@ test.describe('Notebook Tests with CouchDB @couchdb', () => {
         // 2) The shared worker event from 👆 POST request
         // 3) Timestamp update on entry
         // 4) The shared worker event from 👆 POST request
-        addingNotebookElementsRequests = [];
-        await page.locator('text=To start a new entry, click here or drag and drop any object').click();
-        await page.locator('[aria-label="Notebook Entry Input"] >> nth=3').click();
-        await page.locator('[aria-label="Notebook Entry Input"] >> nth=3').fill(`Fourth Entry`);
-        await page.locator('[aria-label="Notebook Entry Input"] >> nth=3').press('Enter');
+        notebookElementsRequests = [];
+        await nbUtils.enterTextEntry(page, 'Fourth Entry');
         page.waitForLoadState('networkidle');
 
-        expect(filterNonFetchRequests(addingNotebookElementsRequests).length).toBeLessThanOrEqual(4);
+        expect(filterNonFetchRequests(notebookElementsRequests).length).toBeLessThanOrEqual(4);
 
         // Add a fifth entry
         // Network requests are:
@@ -203,28 +152,22 @@ test.describe('Notebook Tests with CouchDB @couchdb', () => {
         // 2) The shared worker event from 👆 POST request
         // 3) Timestamp update on entry
         // 4) The shared worker event from 👆 POST request
-        addingNotebookElementsRequests = [];
-        await page.locator('text=To start a new entry, click here or drag and drop any object').click();
-        await page.locator('[aria-label="Notebook Entry Input"] >> nth=4').click();
-        await page.locator('[aria-label="Notebook Entry Input"] >> nth=4').fill(`Fifth Entry`);
-        await page.locator('[aria-label="Notebook Entry Input"] >> nth=4').press('Enter');
+        notebookElementsRequests = [];
+        await nbUtils.enterTextEntry(page, 'Fifth Entry');
         page.waitForLoadState('networkidle');
 
-        expect(filterNonFetchRequests(addingNotebookElementsRequests).length).toBeLessThanOrEqual(4);
+        expect(filterNonFetchRequests(notebookElementsRequests).length).toBeLessThanOrEqual(4);
 
         // Add a sixth entry
         // 1) Send POST to add new entry
         // 2) The shared worker event from 👆 POST request
         // 3) Timestamp update on entry
         // 4) The shared worker event from 👆 POST request
-        addingNotebookElementsRequests = [];
-        await page.locator('text=To start a new entry, click here or drag and drop any object').click();
-        await page.locator('[aria-label="Notebook Entry Input"] >> nth=5').click();
-        await page.locator('[aria-label="Notebook Entry Input"] >> nth=5').fill(`Sixth Entry`);
-        await page.locator('[aria-label="Notebook Entry Input"] >> nth=5').press('Enter');
+        notebookElementsRequests = [];
+        await nbUtils.enterTextEntry(page, 'Sixth Entry');
         page.waitForLoadState('networkidle');
 
-        expect(filterNonFetchRequests(addingNotebookElementsRequests).length).toBeLessThanOrEqual(4);
+        expect(filterNonFetchRequests(notebookElementsRequests).length).toBeLessThanOrEqual(4);
     });
 
     test('Search tests', async ({ page }) => {
@@ -233,35 +176,21 @@ test.describe('Notebook Tests with CouchDB @couchdb', () => {
             description: 'https://github.com/akhenry/openmct-yamcs/issues/69'
         });
         await page.getByText('Annotations').click();
-        await page.locator('text=To start a new entry, click here or drag and drop any object').click();
-        await page.locator('[aria-label="Notebook Entry Input"]').click();
-        await page.locator('[aria-label="Notebook Entry Input"]').fill(`First Entry`);
-        await page.locator('[aria-label="Notebook Entry Input"]').press('Enter');
+        await nbUtils.enterTextEntry(page, 'First Entry');
 
         // Add three tags
-        await page.hover(`button:has-text("Add Tag")`);
-        await page.locator(`button:has-text("Add Tag")`).click();
-        await page.locator('[placeholder="Type to select tag"]').click();
-        await page.locator('[aria-label="Autocomplete Options"] >> text=Science').click();
-        await page.waitForSelector('[aria-label="Tag"]:has-text("Science")');
-
-        await page.hover(`button:has-text("Add Tag")`);
-        await page.locator(`button:has-text("Add Tag")`).click();
-        await page.locator('[placeholder="Type to select tag"]').click();
-        await page.locator('[aria-label="Autocomplete Options"] >> text=Drilling').click();
-        await page.waitForSelector('[aria-label="Tag"]:has-text("Drilling")');
-
-        await page.hover(`button:has-text("Add Tag")`);
-        await page.locator(`button:has-text("Add Tag")`).click();
-        await page.locator('[placeholder="Type to select tag"]').click();
-        await page.locator('[aria-label="Autocomplete Options"] >> text=Driving').click();
-        await page.waitForSelector('[aria-label="Tag"]:has-text("Driving")');
+        await addTagAndAwaitNetwork(page, 'Science');
+        await addTagAndAwaitNetwork(page, 'Drilling');
+        await addTagAndAwaitNetwork(page, 'Driving');
 
         await page.locator('[aria-label="OpenMCT Search"] input[type="search"]').click();
+        //Partial match for "Science" should only return Science
         await page.locator('[aria-label="OpenMCT Search"] input[type="search"]').fill('Sc');
         await expect(page.locator('[aria-label="Search Result"]').first()).toContainText("Science");
         await expect(page.locator('[aria-label="Search Result"]').first()).not.toContainText("Driving");
+        await expect(page.locator('[aria-label="Search Result"]').first()).not.toContainText("Drilling");
 
+        //Searching for a tag which does not exist should return an empty result
         await page.locator('[aria-label="OpenMCT Search"] input[type="search"]').click();
         await page.locator('[aria-label="OpenMCT Search"] input[type="search"]').fill('Xq');
         await expect(page.locator('text=No results found')).toBeVisible();
@@ -275,3 +204,40 @@ function filterNonFetchRequests(requests) {
         return (request.resourceType() === 'fetch');
     });
 }
+
+/**
+ * Add a tag to a notebook entry by providing a tagName.
+ * Reduces indeterminism by waiting until all necessary requests are completed.
+ * @param {import('@playwright/test').Page} page
+ * @param {string} tagName
+ */
+async function addTagAndAwaitNetwork(page, tagName) {
+    await page.hover(`button:has-text("Add Tag")`);
+    await page.locator(`button:has-text("Add Tag")`).click();
+    await page.locator('[placeholder="Type to select tag"]').click();
+    await Promise.all([
+        // Waits for the next request with the specified url
+        page.waitForRequest('**/openmct/_all_docs?include_docs=true'),
+        // Triggers the request
+        page.locator(`[aria-label="Autocomplete Options"] >> text=${tagName}`).click(),
+        expect(page.locator(`[aria-label="Tag"]:has-text("${tagName}")`)).toBeVisible()
+    ]);
+    await page.waitForLoadState('networkidle');
+}
+
+/**
+ * Remove a tag to a notebook entry by providing a tagName.
+ * Reduces indeterminism by waiting until all necessary requests are completed.
+ * @param {import('@playwright/test').Page} page
+ * @param {string} tagName
+ */
+async function removeTagAndAwaitNetwork(page, tagName) {
+    await page.hover(`[aria-label="Tag"]:has-text("${tagName}")`);
+    await Promise.all([
+        page.locator(`[aria-label="Remove tag ${tagName}"]`).click(),
+        //With this pattern, we're awaiting the response but asserting on the request payload.
+        page.waitForResponse(resp => resp.request().postData().includes(`"_deleted":true`) && resp.status() === 201)
+    ]);
+    await expect(page.locator(`[aria-label="Tag"]:has-text("${tagName}")`)).toBeHidden();
+    await page.waitForLoadState('networkidle');
+}

package.json

@@ -5,6 +5,7 @@
   "devDependencies": {
     "@babel/eslint-parser": "7.18.9",
     "@braintree/sanitize-url": "6.0.2",
+    "@deploysentinel/playwright": "0.3.3",
     "@percy/cli": "1.21.0",
     "@percy/playwright": "1.0.4",
     "@playwright/test": "1.29.0",
@@ -72,7 +73,7 @@
     "webpack-merge": "5.8.0"
   },
   "scripts": {
-    "clean": "rm -rf ./dist ./node_modules ./package-lock.json",
+    "clean": "rm -rf ./dist ./node_modules ./package-lock.json ./coverage ./html-test-results ./test-results ./.nyc_output ",
     "clean-test-lint": "npm run clean; npm install; npm run test; npm run lint",
     "start": "npx webpack serve --config ./.webpack/webpack.dev.js",
     "start:coverage": "npx webpack serve --config ./.webpack/webpack.coverage.js",