Add script to delete annotations (#7069)

* add script

* add package.json and script to delete annotations

* amend help

* fix issues

* added explicit runs

* add design document and index creation to script

* update tests to wait for url to change

* i think we can remove this deprecated function now

e2e/appActions.js

@@ -384,11 +384,13 @@ async function setTimeConductorMode(page, isFixedTimespan = true) {
   // Click 'mode' button
   await page.getByRole('button', { name: 'Time Conductor Mode', exact: true }).click();
   await page.getByRole('button', { name: 'Time Conductor Mode Menu' }).click();
-  // Switch time conductor mode
+  // Switch time conductor mode. Note, need to wait here for URL to update as the router is debounced.
   if (isFixedTimespan) {
     await page.getByRole('menuitem', { name: /Fixed Timespan/ }).click();
+    await page.waitForURL(/tc\.mode=fixed/);
   } else {
     await page.getByRole('menuitem', { name: /Real-Time/ }).click();
+    await page.waitForURL(/tc\.mode=local/);
   }
 }
 

e2e/tests/functional/plugins/displayLayout/displayLayout.e2e.spec.js

@@ -263,7 +263,10 @@ test.describe('Display Layout', () => {
     await setFixedTimeMode(page);
     // Create another Sine Wave Generator
     const anotherSineWaveObject = await createDomainObjectWithDefaults(page, {
-      type: 'Sine Wave Generator'
+      type: 'Sine Wave Generator',
+      customParameters: {
+        '[aria-label="Data Rate (hz)"]': '0.01'
+      }
     });
     // Create a Display Layout
     await createDomainObjectWithDefaults(page, {
@@ -306,7 +309,8 @@ test.describe('Display Layout', () => {
     // Time to inspect some network traffic
     let networkRequests = [];
     page.on('request', (request) => {
-      const searchRequest = request.url().endsWith('_find');
+      const searchRequest =
+        request.url().endsWith('_find') || request.url().includes('by_keystring');
       const fetchRequest = request.resourceType() === 'fetch';
       if (searchRequest && fetchRequest) {
         networkRequests.push(request);
@@ -322,6 +326,7 @@ test.describe('Display Layout', () => {
     expect(networkRequests.length).toBe(1);
 
     await setRealTimeMode(page);
+
     networkRequests = [];
 
     await page.reload();

e2e/tests/functional/plugins/timeConductor/timeConductor.e2e.spec.js

@@ -153,7 +153,6 @@ test.describe('Time conductor input fields real-time mode', () => {
     await expect(page.locator('.c-compact-tc__setting-value.icon-plus')).toContainText('00:00:01');
 
     // Verify url parameters persist after mode switch
-    await page.waitForNavigation({ waitUntil: 'networkidle' });
     expect(page.url()).toContain(`startDelta=${startDelta}`);
     expect(page.url()).toContain(`endDelta=${endDelta}`);
   });

e2e/tests/functional/search.e2e.spec.js

@@ -175,7 +175,8 @@ test.describe('Grand Search', () => {
 
     let networkRequests = [];
     page.on('request', (request) => {
-      const searchRequest = request.url().endsWith('_find');
+      const searchRequest =
+        request.url().endsWith('_find') || request.url().includes('by_keystring');
       const fetchRequest = request.resourceType() === 'fetch';
       if (searchRequest && fetchRequest) {
         networkRequests.push(request);

src/plugins/persistence/couch/README.md

@@ -152,6 +152,26 @@ sh ./src/plugins/persistence/couch/replace-localstorage-with-couchdb-indexhtml.s
 4. Look at the 'JSON' tab and ensure you can see the specific object you created above.
 5. All done! 🏆
 
+# Maintenance
+
+One can delete annotations by running inside this directory (i.e., `src/plugins/persistence/couch`):
+```
+npm run deleteAnnotations:openmct:PIXEL_SPATIAL
+```
+
+will delete all image tags.
+
+```
+npm run deleteAnnotations:openmct 
+```
+
+will delete all tags. 
+
+```
+npm run deleteAnnotations:openmct -- --help
+```
+
+will print help options.
 # Search Performance
 
 For large Open MCT installations, it may be helpful to add additional CouchDB capabilities to bear to improve performance.
@@ -159,7 +179,7 @@ For large Open MCT installations, it may be helpful to add additional CouchDB ca
 ## Indexing
 Indexing the `model.type` field in CouchDB can benefit the performance of queries significantly, particularly if there are a large number of documents in the database. An index can accelerate annotation searches by reducing the number of documents that the database needs to examine.
 
-To create an index for `model.type`, you can use the following payload:
+To create an index for `model.type`, you can use the following payload [using the API](https://docs.couchdb.org/en/stable/api/database/find.html#post--db-_index):
 
 ```json
 {
@@ -177,7 +197,7 @@ You can find more detailed information about indexing in CouchDB in the [officia
 
 ## Design Documents
 
-We can also add a design document for retrieving domain objects for specific tags:
+We can also add a design document [through the API](https://docs.couchdb.org/en/stable/api/ddoc/common.html#put--db-_design-ddoc) for retrieving domain objects for specific tags:
 
 ```json
 {

src/plugins/persistence/couch/package.json

@@ -0,0 +1,18 @@
+{
+    "name": "openmct-couch-plugin",
+    "version": "1.0.0",
+    "description": "CouchDB persistence plugin for Open MCT",
+    "dependencies": {
+        "@cloudant/couchbackup": "2.9.9"
+    },
+    "scripts": {
+        "backup:openmct": "npx couchbackup -u http://admin:password@127.0.0.1:5984/ -d openmct -o openmct-couch-backup.txt",
+        "restore:openmct": "cat openmct-couch-backup.txt | npx couchrestore -u http://admin:password@127.0.0.1:5984/ -d openmct",
+        "deleteAnnotations:openmct": "node scripts/deleteAnnotations.js $*",
+        "deleteAnnotations:openmct:NOTEBOOK": "node scripts/deleteAnnotations.js -- --annotationType NOTEBOOK",
+        "deleteAnnotations:openmct:GEOSPATIAL": "node scripts/deleteAnnotations.js -- --annotationType GEOSPATIAL",
+        "deleteAnnotations:openmct:PIXEL_SPATIAL": "node scripts/deleteAnnotations.js -- --annotationType PIXEL_SPATIAL",
+        "deleteAnnotations:openmct:TEMPORAL": "node scripts/deleteAnnotations.js -- --annotationType TEMPORAL",
+        "deleteAnnotations:openmct:PLOT_SPATIAL": "node scripts/deleteAnnotations.js -- --annotationType PLOT_SPATIAL"
+    }
+}

src/plugins/persistence/couch/scripts/deleteAnnotations.js

@@ -0,0 +1,190 @@
+#!/usr/bin/env node
+
+/*****************************************************************************
+ * Open MCT, Copyright (c) 2014-2023, United States Government
+ * as represented by the Administrator of the National Aeronautics and Space
+ * Administration. All rights reserved.
+ *
+ * Open MCT is licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * Open MCT includes source code licensed under additional open source
+ * licenses. See the Open Source Licenses file (LICENSES.md) included with
+ * this source code distribution or the Licensing information page available
+ * at runtime from the About dialog for additional information.
+ *****************************************************************************/
+const process = require('process');
+
+async function main() {
+  try {
+    const { annotationType, serverUrl, databaseName, helpRequested, username, password } =
+      processArguments();
+    if (helpRequested) {
+      return;
+    }
+    const docsToDelete = await gatherDocumentsForDeletion({
+      serverUrl,
+      databaseName,
+      annotationType,
+      username,
+      password
+    });
+    const deletedDocumentCount = await performBulkDelete({
+      docsToDelete,
+      serverUrl,
+      databaseName,
+      username,
+      password
+    });
+    console.log(
+      `Deleted ${deletedDocumentCount} document${deletedDocumentCount === 1 ? '' : 's'}.`
+    );
+  } catch (error) {
+    console.error(`Error: ${error.message}`);
+  }
+}
+
+const ANNOTATION_TYPES = Object.freeze({
+  NOTEBOOK: 'NOTEBOOK',
+  GEOSPATIAL: 'GEOSPATIAL',
+  PIXEL_SPATIAL: 'PIXEL_SPATIAL',
+  TEMPORAL: 'TEMPORAL',
+  PLOT_SPATIAL: 'PLOT_SPATIAL'
+});
+
+function processArguments() {
+  const args = process.argv.slice(2);
+  let annotationType;
+  let databaseName = 'openmct'; // default db name to "openmct"
+  let serverUrl = new URL('http://127.0.0.1:5984'); // default db name to "openmct"
+  let helpRequested = false;
+
+  args.forEach((val, index) => {
+    switch (val) {
+      case '--help':
+        console.log(
+          'Usage: deleteAnnotations.js [--annotationType type] [--dbName name] <CouchDB URL> \nFor authentication, set the environment variables COUCHDB_USERNAME and COUCHDB_PASSWORD. \n'
+        );
+        console.log('Annotation types: ', Object.keys(ANNOTATION_TYPES).join(', '));
+        helpRequested = true;
+        break;
+      case '--annotationType':
+        annotationType = args[index + 1];
+        if (!Object.values(ANNOTATION_TYPES).includes(annotationType)) {
+          throw new Error(`Invalid annotation type: ${annotationType}`);
+        }
+        break;
+      case '--dbName':
+        databaseName = args[index + 1];
+        break;
+      case '--serverUrl':
+        serverUrl = new URL(args[index + 1]);
+        break;
+    }
+  });
+
+  let username = process.env.COUCHDB_USERNAME || '';
+  let password = process.env.COUCHDB_PASSWORD || '';
+
+  return {
+    annotationType,
+    serverUrl,
+    databaseName,
+    helpRequested,
+    username,
+    password
+  };
+}
+
+async function gatherDocumentsForDeletion({
+  serverUrl,
+  databaseName,
+  annotationType,
+  username,
+  password
+}) {
+  const baseUrl = `${serverUrl.href}${databaseName}/_find`;
+  let bookmark = null;
+  let docsToDelete = [];
+  let hasMoreDocs = true;
+
+  const body = {
+    selector: {
+      _id: { $gt: null },
+      'model.type': 'annotation'
+    },
+    fields: ['_id', '_rev'],
+    limit: 1000
+  };
+
+  if (annotationType !== undefined) {
+    body.selector['model.annotationType'] = annotationType;
+  }
+
+  const findOptions = {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify(body)
+  };
+
+  if (username && password) {
+    findOptions.headers.Authorization = `Basic ${btoa(`${username}:${password}`)}`;
+  }
+
+  while (hasMoreDocs) {
+    if (bookmark) {
+      body.bookmark = bookmark;
+    }
+
+    const res = await fetch(baseUrl, findOptions);
+
+    if (!res.ok) {
+      throw new Error(`Server responded with status: ${res.status}`);
+    }
+
+    const findResult = await res.json();
+
+    bookmark = findResult.bookmark;
+    docsToDelete = [...docsToDelete, ...findResult.docs];
+
+    // check if we got less than limit, set hasMoreDocs to false
+    hasMoreDocs = findResult.docs.length === body.limit;
+  }
+
+  return docsToDelete;
+}
+
+async function performBulkDelete({ docsToDelete, serverUrl, databaseName, username, password }) {
+  docsToDelete.forEach((doc) => (doc._deleted = true));
+
+  const deleteOptions = {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify({ docs: docsToDelete })
+  };
+
+  if (username && password) {
+    deleteOptions.headers.Authorization = `Basic ${btoa(`${username}:${password}`)}`;
+  }
+
+  const response = await fetch(`${serverUrl.href}${databaseName}/_bulk_docs`, deleteOptions);
+  if (!response.ok) {
+    throw new Error('Failed with status code: ' + response.status);
+  }
+
+  return docsToDelete.length;
+}
+
+main();

src/plugins/persistence/couch/setup-couchdb.sh

@@ -96,6 +96,72 @@ create_replicator_table() {
     fi
 }
 
+add_index_and_views() {
+    echo "Adding index and views to $OPENMCT_DATABASE_NAME database"
+
+    # Add type_tags_index
+    response=$(curl --silent --user "${CURL_USERPASS_ARG}" --request POST "$COUCH_BASE_LOCAL"/"$OPENMCT_DATABASE_NAME"/_index/\
+    --header 'Content-Type: application/json' \
+    --data '{
+        "index": {
+            "fields": ["model.type", "model.tags"]
+        },
+        "name": "type_tags_index",
+        "type": "json"
+    }')
+
+    if [[ $response =~ "\"result\":\"created\"" ]]; then
+        echo "Successfully created type_tags_index"
+    elif [[ $response =~ "\"result\":\"exists\"" ]]; then
+        echo "type_tags_index already exists, skipping creation"
+    else
+        echo "Unable to create type_tags_index"
+        echo $response
+    fi
+
+    # Add annotation_tags_index
+    response=$(curl  --silent --user "${CURL_USERPASS_ARG}" --request PUT "$COUCH_BASE_LOCAL"/"$OPENMCT_DATABASE_NAME"/_design/annotation_tags_index \
+    --header 'Content-Type: application/json' \
+    --data '{
+        "_id": "_design/annotation_tags_index",
+        "views": {
+            "by_tags": {
+                "map": "function (doc) { if (doc.model && doc.model.type === '\''annotation'\'' && doc.model.tags) { doc.model.tags.forEach(function (tag) { emit(tag, doc._id); }); } }"
+            }
+        }
+    }')
+
+    if [[ $response =~ "\"ok\":true" ]]; then
+        echo "Successfully created annotation_tags_index"
+    elif [[ $response =~ "\"error\":\"conflict\"" ]]; then
+        echo "annotation_tags_index already exists, skipping creation"
+    else
+        echo "Unable to create annotation_tags_index"
+        echo $response
+    fi
+
+    # Add annotation_keystring_index
+    response=$(curl --silent --user "${CURL_USERPASS_ARG}" --request PUT "$COUCH_BASE_LOCAL"/"$OPENMCT_DATABASE_NAME"/_design/annotation_keystring_index \
+    --header 'Content-Type: application/json' \
+    --data '{
+        "_id": "_design/annotation_keystring_index",
+        "views": {
+            "by_keystring": {
+                "map": "function (doc) { if (doc.model && doc.model.type === '\''annotation'\'' && doc.model.targets) { doc.model.targets.forEach(function(target) { if(target.keyString) { emit(target.keyString, doc._id); } }); } }"
+            }
+        }
+    }')
+
+    if [[ $response =~ "\"ok\":true" ]]; then
+        echo "Successfully created annotation_keystring_index"
+    elif [[ $response =~ "\"error\":\"conflict\"" ]]; then
+        echo "annotation_keystring_index already exists, skipping creation"
+    else
+        echo "Unable to create annotation_keystring_index"
+        echo $response
+    fi
+}
+
 # Main script execution
 
 # Check if the admin user exists; if not, create it.
@@ -145,3 +211,6 @@ if [ "FALSE" == "$(is_cors_enabled)" ]; then
 else
     echo "CORS enabled, nothing to do"
 fi
+
+# Add index and views to the database
+add_index_and_views