[Table] Added ticking to combined historical/real-time table

Don't add duplicate telemetry data

platform/features/conductor/utcTimeSystem/src/UTCTimeSystem.js

@@ -25,7 +25,7 @@ define([
     '../../core/src/timeSystems/LocalClock'
 ], function (TimeSystem, LocalClock) {
     var FIFTEEN_MINUTES = 15 * 60 * 1000,
-        DEFAULT_PERIOD = 1000;
+        DEFAULT_PERIOD = 100;
 
     /**
      * This time system supports UTC dates and provides a ticking clock source.

platform/features/table/res/templates/telemetry-table.html

@@ -5,7 +5,7 @@
         headers="headers"
         rows="rows"
         time-columns="tableController.timeColumns"
-        on-show-cell=""
+        format-cell="formatCell"
         enableFilter="true"
         enableSort="true"
         auto-scroll="autoScroll"

platform/features/table/src/TableConfiguration.js

@@ -66,7 +66,8 @@ define(
                             return {
                                 cssClass: alarm && alarm.cssClass,
                                 text: formatter ? formatter.format(telemetryDatum[metadatum.key])
-                                    : telemetryDatum[metadatum.key]
+                                    : telemetryDatum[metadatum.key],
+                                value: telemetryDatum[metadatum.key]
                             }
                         }
                     });

platform/features/table/src/TelemetryCollection.js

@@ -0,0 +1,114 @@
+/*****************************************************************************
+ * Open MCT, Copyright (c) 2014-2016, United States Government
+ * as represented by the Administrator of the National Aeronautics and Space
+ * Administration. All rights reserved.
+ *
+ * Open MCT is licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * Open MCT includes source code licensed under additional open source
+ * licenses. See the Open Source Licenses file (LICENSES.md) included with
+ * this source code distribution or the Licensing information page available
+ * at runtime from the About dialog for additional information.
+ *****************************************************************************/
+
+define(
+    ['lodash'],
+    function (_) {
+        function TelemetryCollection() {
+            this.telemetry = [];
+            this.sortField = undefined;
+            this.lastBounds = {};
+
+            _.bindAll(this,[
+                'iteratee'
+            ]);
+        }
+
+        TelemetryCollection.prototype.iteratee = function (element) {
+            return _.get(element, this.sortField);
+        };
+
+        TelemetryCollection.prototype.bounds = function (bounds) {
+            var startChanged = this.lastBounds.start !== bounds.start;
+            var endChanged = this.lastBounds.end !== bounds.end;
+            var fromStart = 0;
+            var fromEnd = 0;
+            var discarded = [];
+
+            if (startChanged){
+                var testValue = _.set({}, this.sortField, bounds.start);
+                fromStart = _.sortedIndex(this.telemetry, testValue, this.sortField);
+                discarded = this.telemetry.splice(0, fromStart);
+            }
+            if (endChanged) {
+                var testValue = _.set({}, this.sortField, bounds.end);
+                fromEnd = _.sortedLastIndex(this.telemetry, testValue, this.sortField);
+                discarded = discarded.concat(this.telemetry.splice(fromEnd, this.telemetry.length - fromEnd));
+            }
+            this.lastBounds = bounds;
+            return discarded;
+        };
+
+        TelemetryCollection.prototype.isValid = function (element) {
+            var noBoundsDefined = !this.lastBounds || (!this.lastBounds.start && !this.lastBounds.end);
+            var withinBounds = _.get(element, this.sortField) >= this.lastBounds.start &&
+                _.get(element, this.sortField) <= this.lastBounds.end;
+
+            return noBoundsDefined || withinBounds;
+        };
+
+        TelemetryCollection.prototype.add = function (element) {
+            //console.log('data: ' + element.Time.value);
+            if (this.isValid(element)){
+                // Going to check for duplicates. Bound the search problem to
+                // elements around the given time. Use sortedIndex because it
+                // employs a binary search which is O(log n). Can use binary search
+                // based on time stamp because the array is guaranteed ordered due
+                // to sorted insertion.
+
+                var isDuplicate = false;
+                var startIx = _.sortedIndex(this.telemetry, element, this.sortField);
+
+                if (startIx !== this.telemetry.length) {
+                    var endIx = _.sortedLastIndex(this.telemetry, element, this.sortField);
+
+                    // Create an array of potential dupes, based on having the
+                    // same time stamp
+                    var potentialDupes = this.telemetry.slice(startIx, endIx + 1);
+                    // Search potential dupes for exact dupe
+                    isDuplicate = _.findIndex(potentialDupes, _.isEqual.bind(undefined, element)) > -1;
+                }
+
+                if (!isDuplicate) {
+                    this.telemetry.splice(startIx, 0, element);
+                    return true;
+                } else {
+                    return false;
+                }
+
+            } else {
+                return false;
+            }
+        };
+
+        TelemetryCollection.prototype.clear = function () {
+            this.telemetry = undefined;
+        };
+
+        TelemetryCollection.prototype.sort = function (sortField){
+            this.sortField = sortField;
+            this.telemetry = _.sortBy(this.telemetry, this.iteratee);
+        };
+
+        return TelemetryCollection;
+    }
+);

platform/features/table/src/controllers/MCTTableController.js

@@ -35,7 +35,7 @@ define(
                 'changeTimeSystem',
                 'scrollToBottom',
                 'addRow',
-                'removeRow',
+                'removeRows',
                 'onScroll',
                 'firstVisible',
                 'lastVisible',
@@ -126,7 +126,7 @@ define(
              * Listen for rows added individually (eg. for real-time tables)
              */
             $scope.$on('add:row', this.addRow);
-            $scope.$on('remove:row', this.removeRow);
+            $scope.$on('remove:rows', this.removeRows);
 
             /**
              * Populated from the default-sort attribute on MctTable
@@ -229,39 +229,47 @@ define(
          * `remove:row` broadcast event.
          * @private
          */
-        MCTTableController.prototype.removeRow = function (event, rowIndex) {
+        MCTTableController.prototype.removeRows = function (event, rows) {
-            var row = this.$scope.rows[rowIndex],
+            var indexInDisplayRows;
-            // Do a sequential search here. Only way of finding row is by
+            rows.forEach(function (row){
-            // object equality, so array is in effect unsorted.
+                // Do a sequential search here. Only way of finding row is by
+                // object equality, so array is in effect unsorted.
                 indexInDisplayRows = this.$scope.displayRows.indexOf(row);
-            if (indexInDisplayRows !== -1) {
+                if (indexInDisplayRows !== -1) {
-                this.$scope.displayRows.splice(indexInDisplayRows, 1);
+                    this.$scope.displayRows.splice(indexInDisplayRows, 1);
-                this.setVisibleRows();
+                }
-            }
+            }, this);
+
+            this.$scope.sizingRow = this.buildLargestRow([this.$scope.sizingRow].concat(rows));
+
+            this.setElementSizes();
+            this.setVisibleRows()
+                .then(function () {
+                    if (this.$scope.autoScroll) {
+                        this.scrollToBottom();
+                    }
+                }.bind(this));
+
         };
 
         /**
          * @private
          */
         MCTTableController.prototype.onScroll = function (event) {
-                if (!this.scrolling) {
+            requestAnimationFrame(function () {
-                    this.scrolling = true;
+                this.setVisibleRows();
+                this.digest();
 
-                    requestAnimationFrame(function () {
+                // If user scrolls away from bottom, disable auto-scroll.
-                        this.setVisibleRows();
+                // Auto-scroll will be re-enabled if user scrolls to bottom again.
-                        this.digest();
+                if (this.scrollable[0].scrollTop <
-
+                    (this.scrollable[0].scrollHeight - this.scrollable[0].offsetHeight) - 20) {
-                        // If user scrolls away from bottom, disable auto-scroll.
+                    this.$scope.autoScroll = false;
-                        // Auto-scroll will be re-enabled if user scrolls to bottom again.
+                } else {
-                        if (this.scrollable[0].scrollTop <
+                    this.$scope.autoScroll = true;
-                            (this.scrollable[0].scrollHeight - this.scrollable[0].offsetHeight) - 20) {
-                            this.$scope.autoScroll = false;
-                        } else {
-                            this.$scope.autoScroll = true;
-                        }
-                        this.scrolling = false;
-                    }.bind(this));
                 }
+                this.scrolling = false;
+            }.bind(this));
         };
 
         /**
@@ -339,13 +347,19 @@ define(
                     this.$scope.visibleRows[0].rowIndex === start &&
                     this.$scope.visibleRows[this.$scope.visibleRows.length - 1]
                         .rowIndex === end) {
-
+                    return this.digest();
-                    return Promise.resolve(); // don't update if no changes are required.
+                    //return Promise.resolve(); // don't update if no changes are required.
                 }
             }
             //Set visible rows from display rows, based on calculated offset.
             this.$scope.visibleRows = this.$scope.displayRows.slice(start, end)
                 .map(function (row, i) {
+/*                    var formattedRow = JSON.parse(JSON.stringify(row));
+                    if (self.$scope.formatCell) {
+                        Object.keys(formattedRow).forEach(function (header) {
+                            formattedRow[header].text = self.$scope.formatCell(header, row[header].text);
+                        });
+                    } */
                     return {
                         rowIndex: start + i,
                         offsetY: ((start + i) * self.$scope.rowHeight) +

platform/features/table/src/controllers/TelemetryTableController.js

@@ -27,10 +27,11 @@
 define(
     [
         '../TableConfiguration',
-        '../../../../../src/api/objects/object-utils'
+        '../../../../../src/api/objects/object-utils',
+        '../TelemetryCollection'
 
     ],
-    function (TableConfiguration, objectUtils) {
+    function (TableConfiguration, objectUtils, TelemetryCollection) {
 
         /**
          * The TableController is responsible for getting data onto the page
@@ -62,6 +63,7 @@ define(
                 openmct);
             this.lastBounds = this.openmct.conductor.bounds();
             this.requestTime = 0;
+            this.telemetry = new TelemetryCollection();
 
             /*
              * Create a new format object from legacy object, and replace it
@@ -136,18 +138,8 @@ define(
             this.openmct.conductor.on('bounds', this.changeBounds);
         };
 
-        TelemetryTableController.prototype.tick = function (bounds) {
-            // Can't do ticking until we change how data is handled
-            // Pass raw values to table, with format function
-
-            /*if (this.$scope.defaultSort) {
-             this.$scope.rows.filter(function (row){
-             return row[]
-             })
-             }*/
-        };
-
         TelemetryTableController.prototype.changeBounds = function (bounds) {
+            //console.log('bounds.end: ' + bounds.end);
             var follow = this.openmct.conductor.follow();
             var isTick = follow &&
                 bounds.start !== this.lastBounds.start &&
@@ -157,10 +149,16 @@ define(
                 (bounds.start !== this.lastBounds.start ||
                 bounds.end !== this.lastBounds.end);
 
+            var discarded = this.telemetry.bounds(bounds);
+
+            if (discarded.length > 0){
+                this.$scope.$broadcast('remove:rows', discarded);
+            }
+
             if (isTick){
                 // Treat it as a realtime tick
                 // Drop old data that falls outside of bounds
-                this.tick(bounds);
+                //this.tick(bounds);
             } else if (isDeltaChange){
                 // No idea...
                 // Historical query for bounds, then tick on
@@ -214,11 +212,13 @@ define(
                 var allColumns = telemetryApi.commonValuesForHints(metadatas, []);
 
                 this.table.populateColumns(allColumns);
-
                 this.timeColumns = telemetryApi.commonValuesForHints(metadatas, ['x']).map(function (metadatum) {
                     return metadatum.name;
                 });
 
+                // For now, use first time field for time conductor
+                this.telemetry.sort(this.timeColumns[0] + '.value');
+
                 this.filterColumns();
 
                 var timeSystem = this.openmct.conductor.timeSystem();
@@ -241,12 +241,13 @@ define(
             var rowData = [];
             var processedObjects = 0;
             var requestTime = this.lastRequestTime = Date.now();
+            var telemetryCollection = this.telemetry;
 
             return new Promise(function (resolve, reject){
-                function finishProcessing(tableRows){
+                function finishProcessing(){
-                    scope.rows = tableRows.concat(scope.rows);
+                    scope.rows = telemetryCollection.telemetry;
                     scope.loading = false;
-                    resolve(tableRows);
+                    resolve(scope.rows);
                 }
 
                 function processData(historicalData, index, limitEvaluator){
@@ -254,13 +255,14 @@ define(
                         processedObjects++;
 
                         if (processedObjects === objects.length) {
-                            finishProcessing(rowData);
+                            finishProcessing();
                         }
                     } else {
-                        rowData = rowData.concat(
+                        historicalData.slice(index, index + this.batchSize)
-                            historicalData.slice(index, index + this.batchSize).map(
+                            .forEach(function (datum) {
-                                this.table.getRowValues.bind(this.table, limitEvaluator))
+                                telemetryCollection.add(this.table.getRowValues(
-                        );
+                                    limitEvaluator, datum));
+                            }.bind(this));
                         this.timeoutHandle = this.$timeout(processData.bind(
                             this,
                             historicalData,
@@ -305,8 +307,12 @@ define(
          */
         TelemetryTableController.prototype.subscribeToNewData = function (objects) {
             var telemetryApi = this.openmct.telemetry;
+            var telemetryCollection = this.telemetry;
             //Set table max length to avoid unbounded growth.
-            var maxRows = 100000;
+            //var maxRows = 100000;
+            var maxRows = Number.MAX_VALUE;
+            var limitEvaluator;
+            var added = false;
 
             this.subscriptions.forEach(function (subscription) {
                 subscription();
@@ -314,15 +320,15 @@ define(
             this.subscriptions = [];
 
             function newData(domainObject, datum) {
-                this.$scope.rows.push(this.table.getRowValues(
+                limitEvaluator = telemetryApi.limitEvaluator(domainObject);
-                    telemetryApi.limitEvaluator(domainObject), datum));
+                added = telemetryCollection.add(this.table.getRowValues(limitEvaluator, datum));
 
                 //Inform table that a new row has been added
                 if (this.$scope.rows.length > maxRows) {
-                    this.$scope.$broadcast('remove:row', 0);
+                    this.$scope.$broadcast('remove:rows', this.$scope.rows[0]);
                     this.$scope.rows.shift();
                 }
-                if (!this.$scope.loading) {
+                if (!this.$scope.loading && added) {
                     this.$scope.$broadcast('add:row',
                         this.$scope.rows.length - 1);
                 }

platform/features/table/src/directives/MCTTable.js

@@ -94,6 +94,7 @@ define(
                 scope: {
                     headers: "=",
                     rows: "=",
+                    formatCell: "=?",
                     enableFilter: "=?",
                     enableSort: "=?",
                     autoScroll: "=?",