From a599d874eaf1c7f2601a63d6980b8bc22a724ab1 Mon Sep 17 00:00:00 2001 From: Scott Bell Date: Tue, 13 Feb 2024 12:10:33 +0100 Subject: [PATCH] working version --- src/plugins/persistence/couch/README.md | 36 +++++- src/plugins/persistence/couch/package.json | 1 + .../couch/scripts/deleteAnnotations.js | 33 ++++- .../couch/scripts/{restore.js => upsert.js} | 116 ++++++++++++++++-- 4 files changed, 167 insertions(+), 19 deletions(-) rename src/plugins/persistence/couch/scripts/{restore.js => upsert.js} (50%) diff --git a/src/plugins/persistence/couch/README.md b/src/plugins/persistence/couch/README.md index 29421d45ff..c3ddab2a74 100644 --- a/src/plugins/persistence/couch/README.md +++ b/src/plugins/persistence/couch/README.md @@ -215,8 +215,41 @@ This will create a root object with the id of `mine` in both namespaces upon loa 5. All done! 🏆 # Maintenance +All the scripts in this section must be run within this directory (i.e., `src/plugins/persistence/couch`) -One can delete annotations by running inside this directory (i.e., `src/plugins/persistence/couch`): +## Backing Up +One can backup a CouchDB installation by running: +``` +npm run backup:openmct +``` +Note you will need to modify `package.json` to ensure the URL and authorization is correct. + +## Restoring to a New CouchDB Database +One can restore to a new (empty) CouchDB database by running +``` +npm run restore:openmct +``` +Note you will need to modify `package.json` to ensure the URL and authorization is correct. + +# Restoring/Updating an Existing CouchDB database +One can restore or update an existing CouchDB database by running: +``` +npm run upsert:openmct -- --dbName SOME_DB_NAME --backupFilename /path/to/backup.json +``` + +Note the backup file is a JSON file generated from the previously mentioned script. Running this +will take every Open MCT object in the backup, and either insert it as new, or if the object already +exists, update it with the backup version. Note this script does not restore design documents or other +non Open MCT objects. + +``` +npm run upsert:openmct -- --help +``` + +will print help options. + +## Deleting Annotations +One can delete annotations by running: ``` npm run deleteAnnotations:openmct:PIXEL_SPATIAL ``` @@ -235,7 +268,6 @@ npm run deleteAnnotations:openmct -- --help will print help options. # Search Performance - For large Open MCT installations, it may be helpful to add additional CouchDB capabilities to bear to improve performance. ## Indexing diff --git a/src/plugins/persistence/couch/package.json b/src/plugins/persistence/couch/package.json index 4e306e6e17..11ee743990 100644 --- a/src/plugins/persistence/couch/package.json +++ b/src/plugins/persistence/couch/package.json @@ -8,6 +8,7 @@ "scripts": { "backup:openmct": "npx couchbackup -u http://admin:password@127.0.0.1:5984/ -d openmct -o openmct-couch-backup.txt", "restore:openmct": "cat openmct-couch-backup.txt | npx couchrestore -u http://admin:password@127.0.0.1:5984/ -d openmct", + "upsert:openmct": "node scripts/upsert.js $*", "deleteAnnotations:openmct": "node scripts/deleteAnnotations.js $*", "deleteAnnotations:openmct:NOTEBOOK": "node scripts/deleteAnnotations.js -- --annotationType NOTEBOOK", "deleteAnnotations:openmct:GEOSPATIAL": "node scripts/deleteAnnotations.js -- --annotationType GEOSPATIAL", diff --git a/src/plugins/persistence/couch/scripts/deleteAnnotations.js b/src/plugins/persistence/couch/scripts/deleteAnnotations.js index 6756d9de2a..c98ee7105d 100755 --- a/src/plugins/persistence/couch/scripts/deleteAnnotations.js +++ b/src/plugins/persistence/couch/scripts/deleteAnnotations.js @@ -37,6 +37,11 @@ async function main() { username, password }); + if (!docsToDelete.length) { + console.info('🤷‍♂️ No annotations found to delete on server'); + return; + } + const deletedDocumentCount = await performBulkDelete({ docsToDelete, serverUrl, @@ -44,8 +49,8 @@ async function main() { username, password }); - console.log( - `Deleted ${deletedDocumentCount} document${deletedDocumentCount === 1 ? '' : 's'}.` + console.info( + `🎉 Deleted ${deletedDocumentCount} document${deletedDocumentCount === 1 ? '' : 's'}.` ); } catch (error) { console.error(`Error: ${error.message}`); @@ -66,14 +71,15 @@ function processArguments() { let databaseName = 'openmct'; // default db name to "openmct" let serverUrl = new URL('http://127.0.0.1:5984'); // default db name to "openmct" let helpRequested = false; + console.debug = () => {}; args.forEach((val, index) => { switch (val) { case '--help': - console.log( + console.info( 'Usage: deleteAnnotations.js [--annotationType type] [--dbName name] \nFor authentication, set the environment variables COUCHDB_USERNAME and COUCHDB_PASSWORD. \n' ); - console.log('Annotation types: ', Object.keys(ANNOTATION_TYPES).join(', ')); + console.info('Annotation types: ', Object.keys(ANNOTATION_TYPES).join(', ')); helpRequested = true; break; case '--annotationType': @@ -88,6 +94,9 @@ function processArguments() { case '--serverUrl': serverUrl = new URL(args[index + 1]); break; + case '--debug': + console.debug = console.log; + break; } }); @@ -141,9 +150,14 @@ async function gatherDocumentsForDeletion({ findOptions.headers.Authorization = `Basic ${btoa(`${username}:${password}`)}`; } + console.info(`🛜 Contacting ${baseUrl} to find annotations to delete`); while (hasMoreDocs) { if (bookmark) { - body.bookmark = bookmark; + console.debug(`Server has more documents to process, fetching more...`); + findOptions.body = JSON.stringify({ + ...body, + bookmark + }); } const res = await fetch(baseUrl, findOptions); @@ -159,7 +173,11 @@ async function gatherDocumentsForDeletion({ // check if we got less than limit, set hasMoreDocs to false hasMoreDocs = findResult.docs.length === body.limit; + console.debug( + `Fetched ${docsToDelete.length} documents so far, and find result has ${findResult.docs.length} documents` + ); } + console.debug(`Found ${docsToDelete.length} existing annotations on ${baseUrl}`); return docsToDelete; } @@ -179,7 +197,10 @@ async function performBulkDelete({ docsToDelete, serverUrl, databaseName, userna deleteOptions.headers.Authorization = `Basic ${btoa(`${username}:${password}`)}`; } - const response = await fetch(`${serverUrl.href}${databaseName}/_bulk_docs`, deleteOptions); + const baseUrl = `${serverUrl.href}${databaseName}/_bulk_docs`; + + console.info(`🛜 Contacting ${baseUrl} to delete ${docsToDelete.length} annotations`); + const response = await fetch(baseUrl, deleteOptions); if (!response.ok) { throw new Error('Failed with status code: ' + response.status); } diff --git a/src/plugins/persistence/couch/scripts/restore.js b/src/plugins/persistence/couch/scripts/upsert.js similarity index 50% rename from src/plugins/persistence/couch/scripts/restore.js rename to src/plugins/persistence/couch/scripts/upsert.js index 7f7b6b1a4e..586bfaa687 100755 --- a/src/plugins/persistence/couch/scripts/restore.js +++ b/src/plugins/persistence/couch/scripts/upsert.js @@ -22,6 +22,7 @@ * at runtime from the About dialog for additional information. *****************************************************************************/ const process = require('process'); +const fs = require('fs').promises; async function main() { try { @@ -30,16 +31,13 @@ async function main() { if (helpRequested) { return; } - const restoredDocumentCount = await performUpsert({ + await performUpsert({ backupFilename, serverUrl, databaseName, username, password }); - console.log( - `Restored ${restoredDocumentCount} document${restoredDocumentCount === 1 ? '' : 's'}.` - ); } catch (error) { console.error(`Error: ${error.message}`); } @@ -48,15 +46,16 @@ async function main() { function processArguments() { const args = process.argv.slice(2); let databaseName = 'openmct'; // default db name to "openmct" - let serverUrl = new URL('http://127.0.0.1:5984'); // default db name to "openmct" + let serverUrl = new URL('http://127.0.0.1:5984/'); // default db name to "openmct" let backupFilename = 'backup.json'; // default backup filename to "backup.json" let helpRequested = false; + console.debug = () => {}; args.forEach((val, index) => { switch (val) { case '--help': console.log( - 'Usage: restore.js [--backupFilename pathToBackupJSON] [--dbName name] \nFor authentication, set the environment variables COUCHDB_USERNAME and COUCHDB_PASSWORD. \n' + 'Usage: restore.js [--backupFilename pathToBackupJSON] [--dbName name] [--debug] \nFor authentication, set the environment variables COUCHDB_USERNAME and COUCHDB_PASSWORD. \n' ); helpRequested = true; break; @@ -65,10 +64,16 @@ function processArguments() { break; case '--serverUrl': serverUrl = new URL(args[index + 1]); + if (!serverUrl.href.endsWith('/')) { + serverUrl.href += '/'; + } break; case '--backupFilename': backupFilename = args[index + 1]; break; + case '--debug': + console.debug = console.log; + break; } }); @@ -110,10 +115,15 @@ async function getExistingDocsIdToRevMap({ serverUrl, databaseName, username, pa if (username && password) { findOptions.headers.Authorization = `Basic ${btoa(`${username}:${password}`)}`; } + console.info(`🛜 Contacting ${baseUrl} to find existing document revisions`); while (hasMoreDocs) { if (bookmark) { - body.bookmark = bookmark; + console.debug(`Server has more documents to process, fetching more...`); + findOptions.body = JSON.stringify({ + ...body, + bookmark + }); } const res = await fetch(baseUrl, findOptions); @@ -129,20 +139,104 @@ async function getExistingDocsIdToRevMap({ serverUrl, databaseName, username, pa // check if we got less than limit, set hasMoreDocs to false hasMoreDocs = findResult.docs.length === body.limit; + console.debug( + `Fetched ${existingDocs.length} documents so far, and find result has ${findResult.docs.length} documents` + ); } + console.debug(`Found ${existingDocs.length} existing documents on ${baseUrl}`); - return existingDocs; + // transform existinDocs to a map of id to rev + const idToRevMap = existingDocs.reduce((acc, doc) => { + acc[doc._id] = doc._rev; + return acc; + }, {}); + + return idToRevMap; +} + +async function prepareBackupDocuments({ backupFilename, existingDocsidToRevMap }) { + // load backup file + const rawBackup = await fs.readFile(backupFilename); + const backupJSON = JSON.parse(rawBackup); + + console.debug(`${backupJSON.length} backup documents found in ${backupFilename}`); + let newDocumentsToAdd = 0; + let existingDocumentsToUpdate = 0; + const docsToRestore = []; + backupJSON.forEach((backupCouchDocument) => { + if (backupCouchDocument.model && backupCouchDocument._id) { + const existingDocRev = existingDocsidToRevMap[backupCouchDocument._id]; + const docToRestore = { + model: backupCouchDocument.model, + _id: backupCouchDocument._id + }; + if (existingDocRev) { + // need to add rev for couchdb to update + docToRestore._rev = existingDocRev; + existingDocumentsToUpdate++; + } else { + newDocumentsToAdd++; + } + docsToRestore.push(docToRestore); + } else { + console.debug(`Skipping non-model document: ${backupCouchDocument._id}`); + } + }); + + return { docsToRestore, newDocumentsToAdd, existingDocumentsToUpdate }; } async function performUpsert({ backupFilename, serverUrl, databaseName, username, password }) { - console.debug(`👾 Contacting ${serverUrl}/${databaseName}`); - const existingDocs = await getExistingDocsIdToRevMap({ + const existingDocsidToRevMap = await getExistingDocsIdToRevMap({ serverUrl, databaseName, username, password }); - console.debug('🐻 Existing docs:', existingDocs); + + const { docsToRestore, newDocumentsToAdd, existingDocumentsToUpdate } = + await prepareBackupDocuments({ + backupFilename, + existingDocsidToRevMap + }); + + const restoreOptions = { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ docs: docsToRestore }) + }; + if (username && password) { + restoreOptions.headers.Authorization = `Basic ${btoa(`${username}:${password}`)}`; + } + const baseUrl = `${serverUrl.href}${databaseName}/_bulk_docs`; + console.info( + `🛜 Contacting ${baseUrl} to add ${newDocumentsToAdd} new documents, and update ${existingDocumentsToUpdate} existing documents` + ); + const restoreResponse = await fetch(baseUrl, restoreOptions); + + if (!restoreResponse.ok) { + throw new Error(`Server responded with status: ${restoreResponse.status}`); + } + + const restoreJsonResult = await restoreResponse.json(); + let restorationErrorCount = 0; + let restorationSuccessCount = 0; + restoreJsonResult.forEach((result) => { + if (result.error) { + console.error(`🚨 ${result.error} restoring document ${result.id} due to: ${result.reason}`); + restorationErrorCount++; + } else { + restorationSuccessCount++; + } + }); + if (restorationErrorCount > 0) { + console.error(`🚨 ${restorationErrorCount} documents failed to restore`); + } + if (restorationSuccessCount > 0) { + console.info(`🎉 ${restorationSuccessCount} documents restored successfully`); + } } main();