working version

src/plugins/persistence/couch/README.md

@@ -215,8 +215,41 @@ This will create a root object with the id of `mine` in both namespaces upon loa
 5. All done! 🏆
 
 # Maintenance
+All the scripts in this section must be run within this directory (i.e., `src/plugins/persistence/couch`)
 
-One can delete annotations by running inside this directory (i.e., `src/plugins/persistence/couch`):
+## Backing Up
+One can backup a CouchDB installation by running:
+```
+npm run backup:openmct
+```
+Note you will need to modify `package.json` to ensure the URL and authorization is correct.
+
+## Restoring to a New CouchDB Database
+One can restore to a new (empty) CouchDB database by running
+```
+npm run restore:openmct
+```
+Note you will need to modify `package.json` to ensure the URL and authorization is correct.
+
+# Restoring/Updating an Existing CouchDB database
+One can restore or update an existing CouchDB database by running:
+```
+npm run upsert:openmct -- --dbName SOME_DB_NAME --backupFilename /path/to/backup.json
+```
+
+Note the backup file is a JSON file generated from the previously mentioned script. Running this
+will take every Open MCT object in the backup, and either insert it as new, or if the object already
+exists, update it with the backup version. Note this script does not restore design documents or other
+non Open MCT objects.
+
+```
+npm run upsert:openmct -- --help
+```
+
+will print help options.
+
+## Deleting Annotations
+One can delete annotations by running:
 ```
 npm run deleteAnnotations:openmct:PIXEL_SPATIAL
 ```
@@ -235,7 +268,6 @@ npm run deleteAnnotations:openmct -- --help
 
 will print help options.
 # Search Performance
-
 For large Open MCT installations, it may be helpful to add additional CouchDB capabilities to bear to improve performance.
 
 ## Indexing

src/plugins/persistence/couch/package.json

@@ -8,6 +8,7 @@
     "scripts": {
         "backup:openmct": "npx couchbackup -u http://admin:password@127.0.0.1:5984/ -d openmct -o openmct-couch-backup.txt",
         "restore:openmct": "cat openmct-couch-backup.txt | npx couchrestore -u http://admin:password@127.0.0.1:5984/ -d openmct",
+        "upsert:openmct": "node scripts/upsert.js $*",
         "deleteAnnotations:openmct": "node scripts/deleteAnnotations.js $*",
         "deleteAnnotations:openmct:NOTEBOOK": "node scripts/deleteAnnotations.js -- --annotationType NOTEBOOK",
         "deleteAnnotations:openmct:GEOSPATIAL": "node scripts/deleteAnnotations.js -- --annotationType GEOSPATIAL",

src/plugins/persistence/couch/scripts/deleteAnnotations.js

@@ -37,6 +37,11 @@ async function main() {
       username,
       password
     });
+    if (!docsToDelete.length) {
+      console.info('🤷‍♂️ No annotations found to delete on server');
+      return;
+    }
+
     const deletedDocumentCount = await performBulkDelete({
       docsToDelete,
       serverUrl,
@@ -44,8 +49,8 @@ async function main() {
       username,
       password
     });
-    console.log(
+    console.info(
-      `Deleted ${deletedDocumentCount} document${deletedDocumentCount === 1 ? '' : 's'}.`
+      `🎉 Deleted ${deletedDocumentCount} document${deletedDocumentCount === 1 ? '' : 's'}.`
     );
   } catch (error) {
     console.error(`Error: ${error.message}`);
@@ -66,14 +71,15 @@ function processArguments() {
   let databaseName = 'openmct'; // default db name to "openmct"
   let serverUrl = new URL('http://127.0.0.1:5984'); // default db name to "openmct"
   let helpRequested = false;
+  console.debug = () => {};
 
   args.forEach((val, index) => {
     switch (val) {
       case '--help':
-        console.log(
+        console.info(
           'Usage: deleteAnnotations.js [--annotationType type] [--dbName name] <CouchDB URL> \nFor authentication, set the environment variables COUCHDB_USERNAME and COUCHDB_PASSWORD. \n'
         );
-        console.log('Annotation types: ', Object.keys(ANNOTATION_TYPES).join(', '));
+        console.info('Annotation types: ', Object.keys(ANNOTATION_TYPES).join(', '));
         helpRequested = true;
         break;
       case '--annotationType':
@@ -88,6 +94,9 @@ function processArguments() {
       case '--serverUrl':
         serverUrl = new URL(args[index + 1]);
         break;
+      case '--debug':
+        console.debug = console.log;
+        break;
     }
   });
 
@@ -141,9 +150,14 @@ async function gatherDocumentsForDeletion({
     findOptions.headers.Authorization = `Basic ${btoa(`${username}:${password}`)}`;
   }
 
+  console.info(`🛜 Contacting ${baseUrl} to find annotations to delete`);
   while (hasMoreDocs) {
     if (bookmark) {
-      body.bookmark = bookmark;
+      console.debug(`Server has more documents to process, fetching more...`);
+      findOptions.body = JSON.stringify({
+        ...body,
+        bookmark
+      });
     }
 
     const res = await fetch(baseUrl, findOptions);
@@ -159,7 +173,11 @@ async function gatherDocumentsForDeletion({
 
     // check if we got less than limit, set hasMoreDocs to false
     hasMoreDocs = findResult.docs.length === body.limit;
+    console.debug(
+      `Fetched ${docsToDelete.length} documents so far, and find result has ${findResult.docs.length} documents`
+    );
   }
+  console.debug(`Found ${docsToDelete.length} existing annotations on ${baseUrl}`);
 
   return docsToDelete;
 }
@@ -179,7 +197,10 @@ async function performBulkDelete({ docsToDelete, serverUrl, databaseName, userna
     deleteOptions.headers.Authorization = `Basic ${btoa(`${username}:${password}`)}`;
   }
 
-  const response = await fetch(`${serverUrl.href}${databaseName}/_bulk_docs`, deleteOptions);
+  const baseUrl = `${serverUrl.href}${databaseName}/_bulk_docs`;
+
+  console.info(`🛜 Contacting ${baseUrl} to delete ${docsToDelete.length} annotations`);
+  const response = await fetch(baseUrl, deleteOptions);
   if (!response.ok) {
     throw new Error('Failed with status code: ' + response.status);
   }

src/plugins/persistence/couch/scripts/upsert.js

@@ -22,6 +22,7 @@
  * at runtime from the About dialog for additional information.
  *****************************************************************************/
 const process = require('process');
+const fs = require('fs').promises;
 
 async function main() {
   try {
@@ -30,16 +31,13 @@ async function main() {
     if (helpRequested) {
       return;
     }
-    const restoredDocumentCount = await performUpsert({
+    await performUpsert({
       backupFilename,
       serverUrl,
       databaseName,
       username,
       password
     });
-    console.log(
-      `Restored ${restoredDocumentCount} document${restoredDocumentCount === 1 ? '' : 's'}.`
-    );
   } catch (error) {
     console.error(`Error: ${error.message}`);
   }
@@ -48,15 +46,16 @@ async function main() {
 function processArguments() {
   const args = process.argv.slice(2);
   let databaseName = 'openmct'; // default db name to "openmct"
-  let serverUrl = new URL('http://127.0.0.1:5984'); // default db name to "openmct"
+  let serverUrl = new URL('http://127.0.0.1:5984/'); // default db name to "openmct"
   let backupFilename = 'backup.json'; // default backup filename to "backup.json"
   let helpRequested = false;
+  console.debug = () => {};
 
   args.forEach((val, index) => {
     switch (val) {
       case '--help':
         console.log(
-          'Usage: restore.js  [--backupFilename pathToBackupJSON] [--dbName name] <CouchDB URL> \nFor authentication, set the environment variables COUCHDB_USERNAME and COUCHDB_PASSWORD. \n'
+          'Usage: restore.js  [--backupFilename pathToBackupJSON] [--dbName name] [--debug] <CouchDB URL> \nFor authentication, set the environment variables COUCHDB_USERNAME and COUCHDB_PASSWORD. \n'
         );
         helpRequested = true;
         break;
@@ -65,10 +64,16 @@ function processArguments() {
         break;
       case '--serverUrl':
         serverUrl = new URL(args[index + 1]);
+        if (!serverUrl.href.endsWith('/')) {
+          serverUrl.href += '/';
+        }
         break;
       case '--backupFilename':
         backupFilename = args[index + 1];
         break;
+      case '--debug':
+        console.debug = console.log;
+        break;
     }
   });
 
@@ -110,10 +115,15 @@ async function getExistingDocsIdToRevMap({ serverUrl, databaseName, username, pa
   if (username && password) {
     findOptions.headers.Authorization = `Basic ${btoa(`${username}:${password}`)}`;
   }
+  console.info(`🛜 Contacting ${baseUrl} to find existing document revisions`);
 
   while (hasMoreDocs) {
     if (bookmark) {
-      body.bookmark = bookmark;
+      console.debug(`Server has more documents to process, fetching more...`);
+      findOptions.body = JSON.stringify({
+        ...body,
+        bookmark
+      });
     }
 
     const res = await fetch(baseUrl, findOptions);
@@ -129,20 +139,104 @@ async function getExistingDocsIdToRevMap({ serverUrl, databaseName, username, pa
 
     // check if we got less than limit, set hasMoreDocs to false
     hasMoreDocs = findResult.docs.length === body.limit;
+    console.debug(
+      `Fetched ${existingDocs.length} documents so far, and find result has ${findResult.docs.length} documents`
+    );
+  }
+  console.debug(`Found ${existingDocs.length} existing documents on ${baseUrl}`);
+
+  // transform existinDocs to a map of id to rev
+  const idToRevMap = existingDocs.reduce((acc, doc) => {
+    acc[doc._id] = doc._rev;
+    return acc;
+  }, {});
+
+  return idToRevMap;
 }
 
-  return existingDocs;
+async function prepareBackupDocuments({ backupFilename, existingDocsidToRevMap }) {
+  // load backup file
+  const rawBackup = await fs.readFile(backupFilename);
+  const backupJSON = JSON.parse(rawBackup);
+
+  console.debug(`${backupJSON.length} backup documents found in ${backupFilename}`);
+  let newDocumentsToAdd = 0;
+  let existingDocumentsToUpdate = 0;
+  const docsToRestore = [];
+  backupJSON.forEach((backupCouchDocument) => {
+    if (backupCouchDocument.model && backupCouchDocument._id) {
+      const existingDocRev = existingDocsidToRevMap[backupCouchDocument._id];
+      const docToRestore = {
+        model: backupCouchDocument.model,
+        _id: backupCouchDocument._id
+      };
+      if (existingDocRev) {
+        // need to add rev for couchdb to update
+        docToRestore._rev = existingDocRev;
+        existingDocumentsToUpdate++;
+      } else {
+        newDocumentsToAdd++;
+      }
+      docsToRestore.push(docToRestore);
+    } else {
+      console.debug(`Skipping non-model document: ${backupCouchDocument._id}`);
+    }
+  });
+
+  return { docsToRestore, newDocumentsToAdd, existingDocumentsToUpdate };
 }
 
 async function performUpsert({ backupFilename, serverUrl, databaseName, username, password }) {
-  console.debug(`👾 Contacting ${serverUrl}/${databaseName}`);
+  const existingDocsidToRevMap = await getExistingDocsIdToRevMap({
-  const existingDocs = await getExistingDocsIdToRevMap({
     serverUrl,
     databaseName,
     username,
     password
   });
-  console.debug('🐻 Existing docs:', existingDocs);
+
+  const { docsToRestore, newDocumentsToAdd, existingDocumentsToUpdate } =
+    await prepareBackupDocuments({
+      backupFilename,
+      existingDocsidToRevMap
+    });
+
+  const restoreOptions = {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify({ docs: docsToRestore })
+  };
+  if (username && password) {
+    restoreOptions.headers.Authorization = `Basic ${btoa(`${username}:${password}`)}`;
+  }
+  const baseUrl = `${serverUrl.href}${databaseName}/_bulk_docs`;
+  console.info(
+    `🛜 Contacting ${baseUrl} to add ${newDocumentsToAdd} new documents, and update ${existingDocumentsToUpdate} existing documents`
+  );
+  const restoreResponse = await fetch(baseUrl, restoreOptions);
+
+  if (!restoreResponse.ok) {
+    throw new Error(`Server responded with status: ${restoreResponse.status}`);
+  }
+
+  const restoreJsonResult = await restoreResponse.json();
+  let restorationErrorCount = 0;
+  let restorationSuccessCount = 0;
+  restoreJsonResult.forEach((result) => {
+    if (result.error) {
+      console.error(`🚨 ${result.error} restoring document ${result.id} due to: ${result.reason}`);
+      restorationErrorCount++;
+    } else {
+      restorationSuccessCount++;
+    }
+  });
+  if (restorationErrorCount > 0) {
+    console.error(`🚨 ${restorationErrorCount} documents failed to restore`);
+  }
+  if (restorationSuccessCount > 0) {
+    console.info(`🎉 ${restorationSuccessCount} documents restored successfully`);
+  }
 }
 
 main();