Embedding images in notebook entries (#7048)

* initial drag drop, wip

* images work as snapshots, but need to disable navigate to actions

* embed image name

* works now with images, need to be refactor so can duplicate code for entries too

* works dropping on entries too

* handle remote images too

* add e2e test

* spelling

* address most PR comments

e2e/tests/functional/plugins/notebook/notebookSnapshots.e2e.spec.js

@@ -24,9 +24,11 @@
 This test suite is dedicated to tests which verify the basic operations surrounding Notebooks.
 */
 
+const fs = require('fs').promises;
 const { test, expect } = require('../../../../pluginFixtures');
-// const { expandTreePaneItemByName, createDomainObjectWithDefaults } = require('../../../../appActions');
+const { createDomainObjectWithDefaults } = require('../../../../appActions');
-// const nbUtils = require('../../../../helper/notebookUtils');
+
+const NOTEBOOK_NAME = 'Notebook';
 
 test.describe('Snapshot Menu tests', () => {
   test.fixme(
@@ -161,3 +163,57 @@ test.describe('Snapshot Container tests', () => {
     }
   );
 });
+
+test.describe('Snapshot image tests', () => {
+  test.beforeEach(async ({ page }) => {
+    //Navigate to baseURL
+    await page.goto('./', { waitUntil: 'domcontentloaded' });
+
+    // Create Notebook
+    await createDomainObjectWithDefaults(page, {
+      type: NOTEBOOK_NAME
+    });
+  });
+
+  test('Can drop an image onto a notebook and create a new entry', async ({ page }) => {
+    const imageData = await fs.readFile('src/images/favicons/favicon-96x96.png');
+    const imageArray = new Uint8Array(imageData);
+    const fileData = Array.from(imageArray);
+
+    const dropTransfer = await page.evaluateHandle((data) => {
+      const dataTransfer = new DataTransfer();
+      const file = new File([new Uint8Array(data)], 'favicon-96x96.png', { type: 'image/png' });
+      dataTransfer.items.add(file);
+      return dataTransfer;
+    }, fileData);
+
+    await page.dispatchEvent('.c-notebook__drag-area', 'drop', { dataTransfer: dropTransfer });
+
+    // be sure that entry was created
+    await expect(page.getByText('favicon-96x96.png')).toBeVisible();
+
+    // click on image (need to click twice to focus)
+    await page.getByRole('img', { name: 'favicon-96x96.png thumbnail' }).click();
+    await page.getByRole('img', { name: 'favicon-96x96.png thumbnail' }).click();
+
+    // expect large image to be displayed
+    await expect(page.getByRole('dialog').getByText('favicon-96x96.png')).toBeVisible();
+
+    await page.getByLabel('Close').click();
+
+    // drop another image onto the entry
+    await page.dispatchEvent('.c-snapshots', 'drop', { dataTransfer: dropTransfer });
+
+    // expect two embedded images now
+    expect(await page.getByRole('img', { name: 'favicon-96x96.png thumbnail' }).count()).toBe(2);
+
+    await page.locator('.c-snapshot.c-ne__embed').first().getByTitle('More options').click();
+
+    await page.getByRole('menuitem', { name: /Remove This Embed/ }).click();
+
+    await page.getByRole('button', { name: 'Ok', exact: true }).click();
+
+    // expect one embedded image now as we deleted the other
+    expect(await page.getByRole('img', { name: 'favicon-96x96.png thumbnail' }).count()).toBe(1);
+  });
+});

src/plugins/notebook/components/Notebook.vue

@@ -165,6 +165,7 @@ import { isNotebookViewType, RESTRICTED_NOTEBOOK_TYPE } from '../notebook-consta
 import {
   addNotebookEntry,
   createNewEmbed,
+  createNewImageEmbed,
   getEntryPosById,
   getNotebookEntries,
   mutateObject,
@@ -615,12 +616,31 @@ export default {
         this.openmct.editor.cancel();
       }
     },
-    async dropOnEntry(event) {
+    async dropOnEntry(dropEvent) {
-      event.preventDefault();
+      dropEvent.preventDefault();
-      event.stopImmediatePropagation();
+      dropEvent.stopImmediatePropagation();
 
-      const snapshotId = event.dataTransfer.getData('openmct/snapshot/id');
+      const localImageDropped = dropEvent.dataTransfer.files?.[0]?.type.includes('image');
-      if (snapshotId.length) {
+      const imageUrl = dropEvent.dataTransfer.getData('URL');
+      const snapshotId = dropEvent.dataTransfer.getData('openmct/snapshot/id');
+      if (localImageDropped) {
+        // local image dropped from disk (file)
+        const imageData = dropEvent.dataTransfer.files[0];
+        const imageEmbed = await createNewImageEmbed(imageData, this.openmct, imageData?.name);
+        this.newEntry(imageEmbed);
+      } else if (imageUrl) {
+        // remote image dropped (URL)
+        try {
+          const response = await fetch(imageUrl);
+          const imageData = await response.blob();
+          const imageEmbed = await createNewImageEmbed(imageData, this.openmct);
+          this.newEntry(imageEmbed);
+        } catch (error) {
+          this.openmct.notifications.alert(`Unable to add image: ${error.message} `);
+          console.error(`Problem embedding remote image`, error);
+        }
+      } else if (snapshotId.length) {
+        // snapshot object
         const snapshot = this.snapshotContainer.getSnapshot(snapshotId);
         this.newEntry(snapshot.embedObject);
         this.snapshotContainer.removeSnapshot(snapshotId);
@@ -631,11 +651,9 @@ export default {
           namespace
         );
         saveNotebookImageDomainObject(this.openmct, notebookImageDomainObject);
-
+      } else {
-        return;
+        // plain domain object
-      }
+        const data = dropEvent.dataTransfer.getData('openmct/domain-object-path');
-
-      const data = event.dataTransfer.getData('openmct/domain-object-path');
         const objectPath = JSON.parse(data);
         const bounds = this.openmct.time.bounds();
         const snapshotMeta = {
@@ -647,6 +665,7 @@ export default {
         const embed = await createNewEmbed(snapshotMeta);
 
         this.newEntry(embed);
+      }
     },
     focusOnEntryId() {
       if (!this.focusEntryId) {

src/plugins/notebook/components/NotebookEmbed.vue

@@ -27,13 +27,13 @@
     @mouseleave="hideToolTip"
   >
     <div v-if="embed.snapshot" class="c-ne__embed__snap-thumb" @click="openSnapshot()">
-      <img :src="thumbnailImage" />
+      <img :src="thumbnailImage" :alt="`${embed.name} thumbnail`" />
     </div>
     <div class="c-ne__embed__info">
       <div class="c-ne__embed__name">
-        <a class="c-ne__embed__link" :class="embed.cssClass" @click="navigateToItemInTime">{{
+        <a class="c-ne__embed__link" :class="embed.cssClass" @click="navigateToItemInTime">
-          embed.name
+          {{ embed.name }}
-        }}</a>
+        </a>
         <button
           class="c-ne__embed__actions c-icon-button icon-3-dots"
           title="More options"
@@ -144,6 +144,7 @@ export default {
         this.menuActions.splice(0, this.menuActions.length, viewSnapshot);
       }
 
+      if (this.embed.domainObject) {
         const navigateToItem = {
           id: 'navigateToItem',
           cssClass: this.embed.cssClass,
@@ -169,6 +170,7 @@ export default {
         };
 
         this.menuActions.push(...[quickView, navigateToItem, navigateToItemInTime]);
+      }
 
       if (!this.isLocked) {
         const removeEmbed = {
@@ -183,6 +185,9 @@ export default {
       }
     },
     async setEmbedObjectPath() {
+      if (!this.embed.domainObject) {
+        return;
+      }
       this.objectPath = await this.openmct.objects.getOriginalPath(
         this.embed.domainObject.identifier
       );
@@ -260,6 +265,11 @@ export default {
       this.openmct.router.navigate(url);
     },
     navigateToItemInTime() {
+      if (!this.embed.historicLink) {
+        // no historic link available
+
+        return;
+      }
       const hash = this.embed.historicLink;
 
       const bounds = this.openmct.time.bounds();

src/plugins/notebook/components/NotebookEntry.vue

@@ -143,7 +143,7 @@ import Moment from 'moment';
 import sanitizeHtml from 'sanitize-html';
 
 import TextHighlight from '../../../utils/textHighlight/TextHighlight.vue';
-import { createNewEmbed, selectEntry } from '../utils/notebook-entries';
+import { createNewEmbed, createNewImageEmbed, selectEntry } from '../utils/notebook-entries';
 import {
   saveNotebookImageDomainObject,
   updateNamespaceOfDomainObject
@@ -359,11 +359,32 @@ export default {
         this.enableEmbedsWrapperScroll = embedsTotalWidth > embedsWrapperLength;
       }
     },
-    async dropOnEntry($event) {
+    async dropOnEntry(dropEvent) {
-      $event.stopImmediatePropagation();
+      dropEvent.stopImmediatePropagation();
 
-      const snapshotId = $event.dataTransfer.getData('openmct/snapshot/id');
+      const localImageDropped = dropEvent.dataTransfer.files?.[0]?.type.includes('image');
-      if (snapshotId.length) {
+      const snapshotId = dropEvent.dataTransfer.getData('openmct/snapshot/id');
+      const imageUrl = dropEvent.dataTransfer.getData('URL');
+      if (localImageDropped) {
+        // local image dropped from disk (file)
+        const imageData = dropEvent.dataTransfer.files[0];
+        const imageEmbed = await createNewImageEmbed(imageData, this.openmct, imageData?.name);
+        this.entry.embeds.push(imageEmbed);
+        this.manageEmbedLayout();
+      } else if (imageUrl) {
+        try {
+          // remote image dropped (URL)
+          const response = await fetch(imageUrl);
+          const imageData = await response.blob();
+          const imageEmbed = await createNewImageEmbed(imageData, this.openmct);
+          this.entry.embeds.push(imageEmbed);
+          this.manageEmbedLayout();
+        } catch (error) {
+          this.openmct.notifications.alert(`Unable to add image: ${error.message} `);
+          console.error(`Problem embedding remote image`, error);
+        }
+      } else if (snapshotId.length) {
+        // snapshot object
         const snapshot = this.snapshotContainer.getSnapshot(snapshotId);
         this.entry.embeds.push(snapshot.embedObject);
         this.snapshotContainer.removeSnapshot(snapshotId);
@@ -375,7 +396,8 @@ export default {
         );
         saveNotebookImageDomainObject(this.openmct, notebookImageDomainObject);
       } else {
-        const data = $event.dataTransfer.getData('openmct/domain-object-path');
+        // plain domain object
+        const data = dropEvent.dataTransfer.getData('openmct/domain-object-path');
         const objectPath = JSON.parse(data);
         await this.addNewEmbed(objectPath);
       }

src/plugins/notebook/utils/notebook-entries.js

@@ -1,6 +1,11 @@
 import { v4 as uuid } from 'uuid';
 
 import objectLink from '../../../ui/mixins/object-link';
+import {
+  createNotebookImageDomainObject,
+  getThumbnailURLFromImageUrl,
+  saveNotebookImageDomainObject
+} from './notebook-image';
 
 async function getUsername(openmct) {
   let username = null;
@@ -115,24 +120,67 @@ export function getHistoricLinkInFixedMode(openmct, bounds, historicLink) {
   return params.join('&');
 }
 
-export async function createNewEmbed(snapshotMeta, snapshot = '') {
+export function createNewImageEmbed(image, openmct, imageName = '') {
-  const { bounds, link, objectPath, openmct } = snapshotMeta;
+  return new Promise((resolve) => {
-  const domainObject = objectPath[0];
+    const reader = new FileReader();
-  const domainObjectType = openmct.types.get(domainObject.type);
+    reader.onloadend = async () => {
+      const base64Data = reader.result;
+      const blobUrl = URL.createObjectURL(image);
+      const imageDomainObject = createNotebookImageDomainObject(base64Data);
+      await saveNotebookImageDomainObject(openmct, imageDomainObject);
+      const imageThumbnailURL = await getThumbnailURLFromImageUrl(blobUrl);
 
-  const cssClass =
+      const snapshot = {
-    domainObjectType && domainObjectType.definition
+        fullSizeImageObjectIdentifier: imageDomainObject.identifier,
+        thumbnailImage: {
+          src: imageThumbnailURL
+        }
+      };
+
+      const embedMetaData = {
+        bounds: openmct.time.bounds(),
+        link: null,
+        objectPath: null,
+        openmct,
+        userImage: true,
+        imageName
+      };
+
+      const createdEmbed = await createNewEmbed(embedMetaData, snapshot);
+      resolve(createdEmbed);
+    };
+
+    reader.readAsDataURL(image);
+  });
+}
+
+export async function createNewEmbed(snapshotMeta, snapshot = '') {
+  const { bounds, link, objectPath, openmct, userImage } = snapshotMeta;
+  let name = null;
+  let type = null;
+  let cssClass = 'icon-object-unknown';
+  let domainObject = null;
+  let historicLink = null;
+  if (objectPath?.length > 0) {
+    domainObject = objectPath[0];
+    const domainObjectType = openmct.types.get(domainObject.type);
+    cssClass = domainObjectType?.definition
       ? domainObjectType.definition.cssClass
       : 'icon-object-unknown';
-  const date = openmct.time.now();
+    name = domainObject.name;
-  const historicLink = link
+    type = domainObject.identifier.key;
+    historicLink = link
       ? getHistoricLinkInFixedMode(openmct, bounds, link)
       : objectLink.computed.objectLink.call({
           objectPath,
           openmct
         });
-  const name = domainObject.name;
+  } else if (userImage) {
-  const type = domainObject.identifier.key;
+    cssClass = 'icon-image';
+    name = snapshotMeta.imageName;
+  }
+
+  const date = openmct.time.now();
   const createdBy = await getUsername(openmct);
 
   return {

src/plugins/notebook/utils/notebook-image.js

@@ -33,7 +33,7 @@ export function getThumbnailURLFromCanvas(canvas, size = DEFAULT_SIZE) {
   return thumbnailCanvas.toDataURL('image/png');
 }
 
-export function getThumbnailURLFromimageUrl(imageUrl, size = DEFAULT_SIZE) {
+export function getThumbnailURLFromImageUrl(imageUrl, size = DEFAULT_SIZE) {
   return new Promise((resolve) => {
     const image = new Image();
 
@@ -43,7 +43,6 @@ export function getThumbnailURLFromimageUrl(imageUrl, size = DEFAULT_SIZE) {
 
     image.onload = function () {
       canvas.getContext('2d').drawImage(image, 0, 0, size.width, size.height);
-
       resolve(canvas.toDataURL('image/png'));
     };
 
@@ -51,22 +50,8 @@ export function getThumbnailURLFromimageUrl(imageUrl, size = DEFAULT_SIZE) {
   });
 }
 
-export function saveNotebookImageDomainObject(openmct, object) {
+export async function saveNotebookImageDomainObject(openmct, object) {
-  return new Promise((resolve, reject) => {
+  await openmct.objects.save(object);
-    openmct.objects
-      .save(object)
-      .then((result) => {
-        if (result) {
-          resolve(object);
-        } else {
-          reject();
-        }
-      })
-      .catch((e) => {
-        console.error(e);
-        reject();
-      });
-  });
 }
 
 export function updateNotebookImageDomainObject(openmct, identifier, fullSizeImage) {

src/plugins/notebook/utils/notebook-migration.js

@@ -1,7 +1,7 @@
 import { mutateObject } from './notebook-entries';
 import {
   createNotebookImageDomainObject,
-  getThumbnailURLFromimageUrl,
+  getThumbnailURLFromImageUrl,
   saveNotebookImageDomainObject,
   updateNamespaceOfDomainObject
 } from './notebook-image';
@@ -33,7 +33,7 @@ export function notebookImageMigration(openmct, domainObject) {
           const snapshot = embed.snapshot;
           const fullSizeImageURL = snapshot.src;
           if (fullSizeImageURL) {
-            const thumbnailImageURL = await getThumbnailURLFromimageUrl(fullSizeImageURL);
+            const thumbnailImageURL = await getThumbnailURLFromImageUrl(fullSizeImageURL);
             const object = createNotebookImageDomainObject(fullSizeImageURL);
             const notebookImageDomainObject = updateNamespaceOfDomainObject(
               object,

src/plugins/notebook/utils/painterroInstance.js

@@ -1,6 +1,6 @@
 import Painterro from 'painterro';
 
-import { getThumbnailURLFromimageUrl } from './notebook-image';
+import { getThumbnailURLFromImageUrl } from './notebook-image';
 
 const DEFAULT_CONFIG = {
   activeColor: '#ff0000',
@@ -63,7 +63,7 @@ export default class PainterroInstance {
       reader.readAsDataURL(url);
       reader.onloadend = async () => {
         const fullSizeImageURL = reader.result;
-        const thumbnailURL = await getThumbnailURLFromimageUrl(fullSizeImageURL);
+        const thumbnailURL = await getThumbnailURLFromImageUrl(fullSizeImageURL);
         const snapshotObject = {
           fullSizeImage: {
             src: fullSizeImageURL,