From ad7d3d642ef386e09540aaf92ad20d355131226a Mon Sep 17 00:00:00 2001 From: Victor Woeltjen Date: Tue, 29 Sep 2015 10:54:14 -0700 Subject: [PATCH 01/19] [Search] Move reindex function Move function used to listen for mutation changes (to trigger reindexing) up in scope, to avoid retaining references to domain objects via closure. nasa/openmctweb#141 Also, includes misc. whitespace normalization (provided by code editor.) --- .../src/services/GenericSearchProvider.js | 88 ++++++++++--------- 1 file changed, 45 insertions(+), 43 deletions(-) diff --git a/platform/search/src/services/GenericSearchProvider.js b/platform/search/src/services/GenericSearchProvider.js index 014d8d7fda..8688fae68a 100644 --- a/platform/search/src/services/GenericSearchProvider.js +++ b/platform/search/src/services/GenericSearchProvider.js @@ -28,13 +28,13 @@ define( [], function () { "use strict"; - + var DEFAULT_MAX_RESULTS = 100, DEFAULT_TIMEOUT = 1000, stopTime; - + /** - * A search service which searches through domain objects in + * A search service which searches through domain objects in * the filetree without using external search implementations. * * @constructor @@ -44,7 +44,7 @@ define( * domain objects can be gotten. * @param {WorkerService} workerService The service which allows * more easy creation of web workers. - * @param {GENERIC_SEARCH_ROOTS} ROOTS An array of the root + * @param {GENERIC_SEARCH_ROOTS} ROOTS An array of the root * domain objects' IDs. */ function GenericSearchProvider($q, $timeout, objectService, workerService, ROOTS) { @@ -57,11 +57,11 @@ define( this.$q = $q; // pendingQueries is a dictionary with the key value pairs st // the key is the timestamp and the value is the promise - + // Tell the web worker to add a domain object's model to its list of items. function indexItem(domainObject) { var message; - + // undefined check if (domainObject && domainObject.getModel) { // Using model instead of whole domain object because @@ -74,15 +74,15 @@ define( worker.postMessage(message); } } - - // Handles responses from the web worker. Namely, the results of - // a search request. + + // Handles responses from the web worker. Namely, the results of + // a search request. function handleResponse(event) { var ids = [], id; - - // If we have the results from a search + + // If we have the results from a search if (event.data.request === 'search') { // Convert the ids given from the web worker into domain objects for (id in event.data.results) { @@ -91,7 +91,7 @@ define( objectService.getObjects(ids).then(function (objects) { var searchResults = [], id; - + // Create searchResult objects for (id in objects) { searchResults.push({ @@ -100,8 +100,8 @@ define( score: event.data.results[id] }); } - - // Resove the promise corresponding to this + + // Resove the promise corresponding to this pendingQueries[event.data.timestamp].resolve({ hits: searchResults, total: event.data.total, @@ -110,27 +110,44 @@ define( }); } } - + // Helper function for getItems(). Indexes the tree. function indexItems(nodes) { + function handleMutation(model) { + if (model && model.composition) { + // If the node was mutated to have children, get the child domain objects + objectService.getObjects(listener.composition).then(function (objectsById) { + var objects = [], + id; + + // Get each of the domain objects in objectsById + for (id in objectsById) { + objects.push(objectsById[id]); + } + + indexItems(objects); + }); + } + } + nodes.forEach(function (node) { var id = node && node.getId && node.getId(); - + // If we have already indexed this item, stop here if (indexed[id]) { return; } - + // Index each item with the web worker indexItem(node); indexed[id] = true; - - + + // If this node has children, index those if (node && node.hasCapability && node.hasCapability('composition')) { // Make sure that this is async, so doesn't block up page $timeout(function () { - // Get the children... + // Get the children... node.useCapability('composition').then(function (children) { $timeout(function () { // ... then index the children @@ -143,41 +160,26 @@ define( }); }, 0); } - - // Watch for changes to this item, in case it gets new children - if (node && node.hasCapability && node.hasCapability('mutation')) { - node.getCapability('mutation').listen(function (listener) { - if (listener && listener.composition) { - // If the node was mutated to have children, get the child domain objects - objectService.getObjects(listener.composition).then(function (objectsById) { - var objects = [], - id; - // Get each of the domain objects in objectsById - for (id in objectsById) { - objects.push(objectsById[id]); - } - - indexItems(objects); - }); - } - }); + // Watch for changes to this item, in case it gets new children + if (node && node.hasCapability && node.hasCapability('mutation')) { + node.getCapability('mutation').listen(handleMutation); } }); } - + // Converts the filetree into a list function getItems() { // Aquire root objects objectService.getObjects(ROOTS).then(function (objectsById) { var objects = [], id; - + // Get each of the domain objects in objectsById for (id in objectsById) { objects.push(objectsById[id]); } - + // Index all of the roots' descendents indexItems(objects); }); @@ -185,7 +187,7 @@ define( worker.onmessage = handleResponse; - // Index the tree's contents once at the beginning + // Index the tree's contents once at the beginning getItems(); } @@ -266,4 +268,4 @@ define( return GenericSearchProvider; } -); \ No newline at end of file +); From 866c8882ca511862c152e46e79bfbd59946191bc Mon Sep 17 00:00:00 2001 From: Victor Woeltjen Date: Tue, 29 Sep 2015 11:47:02 -0700 Subject: [PATCH 02/19] [Search] Listen on a global mutation topic Listen on a global mutation topic to remove the need to retain listeners per domain object. --- .../src/capabilities/MutationCapability.js | 21 +++++++++--- platform/search/bundle.json | 11 +++++-- .../src/services/GenericSearchProvider.js | 32 +++++-------------- 3 files changed, 34 insertions(+), 30 deletions(-) diff --git a/platform/core/src/capabilities/MutationCapability.js b/platform/core/src/capabilities/MutationCapability.js index b9f49ca969..8746b9af64 100644 --- a/platform/core/src/capabilities/MutationCapability.js +++ b/platform/core/src/capabilities/MutationCapability.js @@ -29,7 +29,8 @@ define( function () { "use strict"; - var TOPIC_PREFIX = "mutation:"; + var GENERAL_TOPIC = "mutation", + TOPIC_PREFIX = "mutation:"; // Utility function to overwrite a destination object // with the contents of a source object. @@ -78,7 +79,11 @@ define( * @implements {Capability} */ function MutationCapability(topic, now, domainObject) { - this.mutationTopic = topic(TOPIC_PREFIX + domainObject.getId()); + this.generalMutationTopic = + topic(GENERAL_TOPIC); + this.specificMutationTopic = + topic(TOPIC_PREFIX + domainObject.getId()); + this.now = now; this.domainObject = domainObject; } @@ -115,11 +120,19 @@ define( // mutator function has a temporary copy to work with. var domainObject = this.domainObject, now = this.now, - t = this.mutationTopic, + generalTopic = this.generalMutationTopic, + specificTopic = this.specificMutationTopic, model = domainObject.getModel(), clone = JSON.parse(JSON.stringify(model)), useTimestamp = arguments.length > 1; + function notifyListeners(model) { + // Broadcast a general event... + generalTopic.notify(domainObject); + // ...and also notify listeners watching this specific object. + specificTopic.notify(model); + } + // Function to handle copying values to the actual function handleMutation(mutationResult) { // If mutation result was undefined, just use @@ -136,7 +149,7 @@ define( copyValues(model, result); } model.modified = useTimestamp ? timestamp : now(); - t.notify(model); + notifyListeners(model); } // Report the result of the mutation diff --git a/platform/search/bundle.json b/platform/search/bundle.json index 7ea1536556..4602227c27 100644 --- a/platform/search/bundle.json +++ b/platform/search/bundle.json @@ -45,7 +45,14 @@ "provides": "searchService", "type": "provider", "implementation": "services/GenericSearchProvider.js", - "depends": [ "$q", "$timeout", "objectService", "workerService", "GENERIC_SEARCH_ROOTS" ] + "depends": [ + "$q", + "$timeout", + "objectService", + "workerService", + "topic", + "GENERIC_SEARCH_ROOTS" + ] }, { "provides": "searchService", @@ -61,4 +68,4 @@ } ] } -} \ No newline at end of file +} diff --git a/platform/search/src/services/GenericSearchProvider.js b/platform/search/src/services/GenericSearchProvider.js index 8688fae68a..eba2b7f725 100644 --- a/platform/search/src/services/GenericSearchProvider.js +++ b/platform/search/src/services/GenericSearchProvider.js @@ -47,10 +47,11 @@ define( * @param {GENERIC_SEARCH_ROOTS} ROOTS An array of the root * domain objects' IDs. */ - function GenericSearchProvider($q, $timeout, objectService, workerService, ROOTS) { + function GenericSearchProvider($q, $timeout, objectService, workerService, topic, ROOTS) { var indexed = {}, pendingQueries = {}, - worker = workerService.run('genericSearchWorker'); + worker = workerService.run('genericSearchWorker'), + mutationTopic = topic("mutation"); this.worker = worker; this.pendingQueries = pendingQueries; @@ -113,23 +114,6 @@ define( // Helper function for getItems(). Indexes the tree. function indexItems(nodes) { - function handleMutation(model) { - if (model && model.composition) { - // If the node was mutated to have children, get the child domain objects - objectService.getObjects(listener.composition).then(function (objectsById) { - var objects = [], - id; - - // Get each of the domain objects in objectsById - for (id in objectsById) { - objects.push(objectsById[id]); - } - - indexItems(objects); - }); - } - } - nodes.forEach(function (node) { var id = node && node.getId && node.getId(); @@ -160,11 +144,6 @@ define( }); }, 0); } - - // Watch for changes to this item, in case it gets new children - if (node && node.hasCapability && node.hasCapability('mutation')) { - node.getCapability('mutation').listen(handleMutation); - } }); } @@ -189,6 +168,11 @@ define( // Index the tree's contents once at the beginning getItems(); + + // Re-index items when they are mutated + mutationTopic.listen(function (domainObject) { + indexItems([domainObject]); + }); } /** From fe8543158e78bf1d802c7047e27c7843cacec308 Mon Sep 17 00:00:00 2001 From: Victor Woeltjen Date: Tue, 29 Sep 2015 12:00:05 -0700 Subject: [PATCH 03/19] [Search] Fix reindexing Flag domain objects as non-indexed on mutation to ensure reindexing. Done in the context of nasa/openmctweb#141. --- platform/search/src/services/GenericSearchProvider.js | 1 + 1 file changed, 1 insertion(+) diff --git a/platform/search/src/services/GenericSearchProvider.js b/platform/search/src/services/GenericSearchProvider.js index eba2b7f725..a8e7976c31 100644 --- a/platform/search/src/services/GenericSearchProvider.js +++ b/platform/search/src/services/GenericSearchProvider.js @@ -171,6 +171,7 @@ define( // Re-index items when they are mutated mutationTopic.listen(function (domainObject) { + indexed[domainObject.getId()] = false; indexItems([domainObject]); }); } From 77c399f2a26380c9dead3bb9239bb753ffef33cb Mon Sep 17 00:00:00 2001 From: Victor Woeltjen Date: Tue, 29 Sep 2015 12:19:35 -0700 Subject: [PATCH 04/19] [Search] Don't trigger digest cycles while indexing Avoid triggering digest cycles while indexing; remove extra call to $timeout --- .../search/src/services/GenericSearchProvider.js | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/platform/search/src/services/GenericSearchProvider.js b/platform/search/src/services/GenericSearchProvider.js index a8e7976c31..e05b58ca96 100644 --- a/platform/search/src/services/GenericSearchProvider.js +++ b/platform/search/src/services/GenericSearchProvider.js @@ -126,23 +126,13 @@ define( indexItem(node); indexed[id] = true; - // If this node has children, index those if (node && node.hasCapability && node.hasCapability('composition')) { // Make sure that this is async, so doesn't block up page $timeout(function () { // Get the children... - node.useCapability('composition').then(function (children) { - $timeout(function () { - // ... then index the children - if (children.constructor === Array) { - indexItems(children); - } else { - indexItems([children]); - } - }, 0); - }); - }, 0); + node.useCapability('composition').then(indexItems); + }, 0, false); } }); } From 2979ee90a38dbbb0b197c9e4414cc6a9207ba333 Mon Sep 17 00:00:00 2001 From: Victor Woeltjen Date: Tue, 29 Sep 2015 14:59:49 -0700 Subject: [PATCH 05/19] Revert "[Search] Don't trigger digest cycles while indexing" This reverts commit 4b8a5ac0b257737ee33effc966816afca6c11b94. Performance measurements indicates this is harmful for performance, although this is not well-explained. --- .../search/src/services/GenericSearchProvider.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/platform/search/src/services/GenericSearchProvider.js b/platform/search/src/services/GenericSearchProvider.js index e05b58ca96..a8e7976c31 100644 --- a/platform/search/src/services/GenericSearchProvider.js +++ b/platform/search/src/services/GenericSearchProvider.js @@ -126,13 +126,23 @@ define( indexItem(node); indexed[id] = true; + // If this node has children, index those if (node && node.hasCapability && node.hasCapability('composition')) { // Make sure that this is async, so doesn't block up page $timeout(function () { // Get the children... - node.useCapability('composition').then(indexItems); - }, 0, false); + node.useCapability('composition').then(function (children) { + $timeout(function () { + // ... then index the children + if (children.constructor === Array) { + indexItems(children); + } else { + indexItems([children]); + } + }, 0); + }); + }, 0); } }); } From 0891e15936807a37923c0c2ffe83278eea0851b2 Mon Sep 17 00:00:00 2001 From: Victor Woeltjen Date: Tue, 29 Sep 2015 15:01:09 -0700 Subject: [PATCH 06/19] [Search] Add digest indicator Add digest indicator to example/profiling to aid in diagnosing digest-related performance issues (or ruling out excessive digest cycles as a cause of performance issues.) --- example/profiling/bundle.json | 6 +- example/profiling/src/DigestIndicator.js | 77 ++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 example/profiling/src/DigestIndicator.js diff --git a/example/profiling/bundle.json b/example/profiling/bundle.json index b6090717d2..25c1b10749 100644 --- a/example/profiling/bundle.json +++ b/example/profiling/bundle.json @@ -4,7 +4,11 @@ { "implementation": "WatchIndicator.js", "depends": ["$interval", "$rootScope"] + }, + { + "implementation": "DigestIndicator.js", + "depends": ["$interval", "$rootScope"] } ] } -} \ No newline at end of file +} diff --git a/example/profiling/src/DigestIndicator.js b/example/profiling/src/DigestIndicator.js new file mode 100644 index 0000000000..02fbc7a08b --- /dev/null +++ b/example/profiling/src/DigestIndicator.js @@ -0,0 +1,77 @@ +/***************************************************************************** + * Open MCT Web, Copyright (c) 2014-2015, United States Government + * as represented by the Administrator of the National Aeronautics and Space + * Administration. All rights reserved. + * + * Open MCT Web is licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * Open MCT Web includes source code licensed under additional open source + * licenses. See the Open Source Licenses file (LICENSES.md) included with + * this source code distribution or the Licensing information page available + * at runtime from the About dialog for additional information. + *****************************************************************************/ +/*global define*/ + +define( + [], + function () { + "use strict"; + + /** + * Displays the number of digests that have occurred since the + * indicator was first instantiated. + * @constructor + * @param $interval Angular's $interval + * @implements {Indicator} + */ + function DigestIndicator($interval, $rootScope) { + var digests = 0, + displayed = 0, + start = Date.now(); + + function update() { + var secs = (Date.now() - start) / 1000; + displayed = Math.round(digests / secs); + } + + function increment() { + digests += 1; + } + + $rootScope.$watch(increment); + + // Update state every second + $interval(update, 1000); + + // Provide initial state, too + update(); + + return { + getGlyph: function () { + return "."; + }, + getGlyphClass: function () { + return undefined; + }, + getText: function () { + return displayed + " digests/sec"; + }, + getDescription: function () { + return ""; + } + }; + } + + return DigestIndicator; + + } +); From 77c66053f3964456ae47317eb1d750584161b255 Mon Sep 17 00:00:00 2001 From: Victor Woeltjen Date: Tue, 29 Sep 2015 15:57:08 -0700 Subject: [PATCH 07/19] [Search] Change indexing approach Change indexing approach to more carefully control the rate at which objects are loaded to be indexed, to improve performance. nasa/openmctweb#141. --- platform/search/bundle.json | 2 +- .../src/services/GenericSearchProvider.js | 115 ++++++++---------- 2 files changed, 50 insertions(+), 67 deletions(-) diff --git a/platform/search/bundle.json b/platform/search/bundle.json index 4602227c27..c0dd7e29a9 100644 --- a/platform/search/bundle.json +++ b/platform/search/bundle.json @@ -47,7 +47,7 @@ "implementation": "services/GenericSearchProvider.js", "depends": [ "$q", - "$timeout", + "throttle", "objectService", "workerService", "topic", diff --git a/platform/search/src/services/GenericSearchProvider.js b/platform/search/src/services/GenericSearchProvider.js index a8e7976c31..a2b0fdd407 100644 --- a/platform/search/src/services/GenericSearchProvider.js +++ b/platform/search/src/services/GenericSearchProvider.js @@ -31,6 +31,8 @@ define( var DEFAULT_MAX_RESULTS = 100, DEFAULT_TIMEOUT = 1000, + FLUSH_SIZE = 24, + FLUSH_INTERVAL = 25, stopTime; /** @@ -39,7 +41,7 @@ define( * * @constructor * @param $q Angular's $q, for promise consolidation. - * @param $timeout Angular's $timeout, for delayed function execution. + * @param {Function} throttle a function to throttle function invocations * @param {ObjectService} objectService The service from which * domain objects can be gotten. * @param {WorkerService} workerService The service which allows @@ -47,11 +49,13 @@ define( * @param {GENERIC_SEARCH_ROOTS} ROOTS An array of the root * domain objects' IDs. */ - function GenericSearchProvider($q, $timeout, objectService, workerService, topic, ROOTS) { + function GenericSearchProvider($q, throttle, objectService, workerService, topic, ROOTS) { var indexed = {}, pendingQueries = {}, + toIndex = {}, worker = workerService.run('genericSearchWorker'), - mutationTopic = topic("mutation"); + mutationTopic = topic("mutation"), + scheduleFlush; this.worker = worker; this.pendingQueries = pendingQueries; @@ -59,23 +63,30 @@ define( // pendingQueries is a dictionary with the key value pairs st // the key is the timestamp and the value is the promise - // Tell the web worker to add a domain object's model to its list of items. - function indexItem(domainObject) { - var message; - - // undefined check - if (domainObject && domainObject.getModel) { - // Using model instead of whole domain object because - // it's a JSON object. - message = { - request: 'index', - model: domainObject.getModel(), - id: domainObject.getId() - }; - worker.postMessage(message); - } + function scheduleIdsForIndexing(ids) { + ids.forEach(function (id) { + if (!indexed[id]) { + indexed[id] = true; + toIndex[id] = true; + } + }); + scheduleFlush(); } + // Tell the web worker to add a domain object's model to its list of items. + function indexItem(domainObject) { + var model = domainObject.getModel(); + + worker.postMessage({ + request: 'index', + model: model, + id: domainObject.getId() + }); + + if (Array.isArray(model.composition)) { + scheduleIdsForIndexing(model.composition); + } + } // Handles responses from the web worker. Namely, the results of // a search request. @@ -112,67 +123,39 @@ define( } } - // Helper function for getItems(). Indexes the tree. - function indexItems(nodes) { - nodes.forEach(function (node) { - var id = node && node.getId && node.getId(); + scheduleFlush = throttle(function flush() { + var ids = Object.keys(toIndex).slice(0, FLUSH_SIZE); - // If we have already indexed this item, stop here - if (indexed[id]) { - return; - } - - // Index each item with the web worker - indexItem(node); - indexed[id] = true; - - - // If this node has children, index those - if (node && node.hasCapability && node.hasCapability('composition')) { - // Make sure that this is async, so doesn't block up page - $timeout(function () { - // Get the children... - node.useCapability('composition').then(function (children) { - $timeout(function () { - // ... then index the children - if (children.constructor === Array) { - indexItems(children); - } else { - indexItems([children]); - } - }, 0); - }); - }, 0); - } + // Don't need to look these up next time + ids.forEach(function (id) { + delete toIndex[id]; }); - } - // Converts the filetree into a list - function getItems() { - // Aquire root objects - objectService.getObjects(ROOTS).then(function (objectsById) { - var objects = [], - id; + if (ids.length < 1) { + return; + } - // Get each of the domain objects in objectsById - for (id in objectsById) { - objects.push(objectsById[id]); - } + objectService.getObjects(ids).then(function (objects) { + ids.map(function (id) { + return objects[id]; + }).filter(function (object) { + return object; + }).forEach(indexItem); - // Index all of the roots' descendents - indexItems(objects); + scheduleFlush(); }); - } + }, FLUSH_INTERVAL); worker.onmessage = handleResponse; // Index the tree's contents once at the beginning - getItems(); + scheduleIdsForIndexing(ROOTS); // Re-index items when they are mutated mutationTopic.listen(function (domainObject) { - indexed[domainObject.getId()] = false; - indexItems([domainObject]); + var id = domainObject.getId(); + indexed[id] = false; + scheduleIdsForIndexing([id]); }); } From c2868a4573d8244fd8e8d0c9a54821dbdff39f65 Mon Sep 17 00:00:00 2001 From: Victor Woeltjen Date: Wed, 16 Sep 2015 10:18:57 -0700 Subject: [PATCH 08/19] [Time Conductor] Allow arguments for throttled functions WTD-1515. Ensures that bounds passed in from the time controller get appropriately captured. --- platform/core/src/services/Throttle.js | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/platform/core/src/services/Throttle.js b/platform/core/src/services/Throttle.js index 3d68988d6b..eda6713dec 100644 --- a/platform/core/src/services/Throttle.js +++ b/platform/core/src/services/Throttle.js @@ -36,11 +36,16 @@ define( * * Returns a function that, when invoked, will invoke `fn` after * `delay` milliseconds, only if no other invocations are pending. - * The optional argument `apply` determines whether. + * The optional argument `apply` determines whether or not a + * digest cycle should be triggered. * * The returned function will itself return a `Promise` which will * resolve to the returned value of `fn` whenever that is invoked. * + * In cases where arguments are provided, only the most recent + * set of arguments will be passed on to the throttled function + * at the time it is executed. + * * @returns {Function} * @memberof platform/core */ @@ -56,7 +61,8 @@ define( * @memberof platform/core.Throttle# */ return function (fn, delay, apply) { - var activeTimeout; + var activeTimeout, + args = []; // Clear active timeout, so that next invocation starts // a new one. @@ -64,14 +70,21 @@ define( activeTimeout = undefined; } + // Invoke the function with the latest supplied arguments. + function invoke() { + fn.apply(null, args); + } + // Defaults delay = delay || 0; apply = apply || false; return function () { + // Store arguments from this invocation + args = Array.prototype.slice.apply(arguments, [0]); // Start a timeout if needed if (!activeTimeout) { - activeTimeout = $timeout(fn, delay, apply); + activeTimeout = $timeout(invoke, delay, apply); activeTimeout.then(clearActiveTimeout); } // Return whichever timeout is active (to get From ef527df381fff1765b34b217fe2a1b9375d1e875 Mon Sep 17 00:00:00 2001 From: Victor Woeltjen Date: Wed, 16 Sep 2015 11:04:07 -0700 Subject: [PATCH 09/19] [Time Conductor] Fix throttle bug Fix a timing/ordering issue in throttle which allowed some throttled invocations to be ignored. WTD-1515 --- platform/core/src/services/Throttle.js | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/platform/core/src/services/Throttle.js b/platform/core/src/services/Throttle.js index eda6713dec..60444ad6c4 100644 --- a/platform/core/src/services/Throttle.js +++ b/platform/core/src/services/Throttle.js @@ -61,18 +61,14 @@ define( * @memberof platform/core.Throttle# */ return function (fn, delay, apply) { - var activeTimeout, + var promise, // Promise for the result of throttled function args = []; - // Clear active timeout, so that next invocation starts - // a new one. - function clearActiveTimeout() { - activeTimeout = undefined; - } - - // Invoke the function with the latest supplied arguments. function invoke() { - fn.apply(null, args); + // Clear the active timeout so a new one starts next time. + promise = undefined; + // Invoke the function with the latest supplied arguments. + return fn.apply(null, args); } // Defaults @@ -83,13 +79,10 @@ define( // Store arguments from this invocation args = Array.prototype.slice.apply(arguments, [0]); // Start a timeout if needed - if (!activeTimeout) { - activeTimeout = $timeout(invoke, delay, apply); - activeTimeout.then(clearActiveTimeout); - } + promise = promise || $timeout(invoke, delay, apply); // Return whichever timeout is active (to get // a promise for the results of fn) - return activeTimeout; + return promise; }; }; } From 85ac4a9a320d6ebf963a6fe60541c1110b037d15 Mon Sep 17 00:00:00 2001 From: Victor Woeltjen Date: Tue, 29 Sep 2015 16:35:27 -0700 Subject: [PATCH 10/19] [Search] Log indexing time ...to aid in tuning of generic search parameters. --- platform/search/bundle.json | 1 + platform/search/src/services/GenericSearchProvider.js | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/platform/search/bundle.json b/platform/search/bundle.json index c0dd7e29a9..67dc058ed9 100644 --- a/platform/search/bundle.json +++ b/platform/search/bundle.json @@ -47,6 +47,7 @@ "implementation": "services/GenericSearchProvider.js", "depends": [ "$q", + "$log", "throttle", "objectService", "workerService", diff --git a/platform/search/src/services/GenericSearchProvider.js b/platform/search/src/services/GenericSearchProvider.js index a2b0fdd407..3eb31f255a 100644 --- a/platform/search/src/services/GenericSearchProvider.js +++ b/platform/search/src/services/GenericSearchProvider.js @@ -41,6 +41,7 @@ define( * * @constructor * @param $q Angular's $q, for promise consolidation. + * @param $log Anglar's $log, for logging. * @param {Function} throttle a function to throttle function invocations * @param {ObjectService} objectService The service from which * domain objects can be gotten. @@ -49,12 +50,13 @@ define( * @param {GENERIC_SEARCH_ROOTS} ROOTS An array of the root * domain objects' IDs. */ - function GenericSearchProvider($q, throttle, objectService, workerService, topic, ROOTS) { + function GenericSearchProvider($q, $log, throttle, objectService, workerService, topic, ROOTS) { var indexed = {}, pendingQueries = {}, toIndex = {}, worker = workerService.run('genericSearchWorker'), mutationTopic = topic("mutation"), + indexingStarted = Date.now(), scheduleFlush; this.worker = worker; @@ -132,6 +134,11 @@ define( }); if (ids.length < 1) { + $log.info([ + 'GenericSearch finished indexing after ', + ((Date.now() - indexingStarted) / 1000).toFixed(2), + ' seconds.' + ].join('')); return; } From 78f3f8367e14a38c97ffee98489b5d84114ce347 Mon Sep 17 00:00:00 2001 From: Victor Woeltjen Date: Tue, 29 Sep 2015 18:35:55 -0700 Subject: [PATCH 11/19] [Search] Vary batch size When indexing for generic search, issue new batches of requests as individual requests finish (instead of waiting for whole batches to finish) varying size to keep the number of outstanding requests below some maximum. nasa/openmctweb#141 --- .../src/services/GenericSearchProvider.js | 49 ++++++++++--------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/platform/search/src/services/GenericSearchProvider.js b/platform/search/src/services/GenericSearchProvider.js index 3eb31f255a..a8387b143f 100644 --- a/platform/search/src/services/GenericSearchProvider.js +++ b/platform/search/src/services/GenericSearchProvider.js @@ -31,8 +31,8 @@ define( var DEFAULT_MAX_RESULTS = 100, DEFAULT_TIMEOUT = 1000, - FLUSH_SIZE = 24, - FLUSH_INTERVAL = 25, + MAX_CONCURRENT_REQUESTS = 100, + FLUSH_INTERVAL = 180, stopTime; /** @@ -53,10 +53,11 @@ define( function GenericSearchProvider($q, $log, throttle, objectService, workerService, topic, ROOTS) { var indexed = {}, pendingQueries = {}, - toIndex = {}, + toRequest = {}, worker = workerService.run('genericSearchWorker'), mutationTopic = topic("mutation"), indexingStarted = Date.now(), + pendingRequests = 0, scheduleFlush; this.worker = worker; @@ -69,7 +70,7 @@ define( ids.forEach(function (id) { if (!indexed[id]) { indexed[id] = true; - toIndex[id] = true; + toRequest[id] = true; } }); scheduleFlush(); @@ -125,32 +126,36 @@ define( } } - scheduleFlush = throttle(function flush() { - var ids = Object.keys(toIndex).slice(0, FLUSH_SIZE); - - // Don't need to look these up next time - ids.forEach(function (id) { - delete toIndex[id]; + function requestAndIndex(id) { + delete toRequest[id]; + pendingRequests += 1; + objectService.getObjects([id]).then(function (objects) { + if (objects[id]) { + indexItem(objects[id]); + } + }, function () { + $log.warn("Failed to index domain object " + id); + }).then(function () { + pendingRequests -= 1; + scheduleFlush(); }); + } - if (ids.length < 1) { + scheduleFlush = throttle(function flush() { + var batchSize = + Math.max(MAX_CONCURRENT_REQUESTS - pendingRequests, 0); + + if (Object.keys(toRequest).length + pendingRequests < 1) { $log.info([ 'GenericSearch finished indexing after ', ((Date.now() - indexingStarted) / 1000).toFixed(2), ' seconds.' ].join('')); - return; + } else { + Object.keys(toRequest) + .slice(0, batchSize) + .forEach(requestAndIndex); } - - objectService.getObjects(ids).then(function (objects) { - ids.map(function (id) { - return objects[id]; - }).filter(function (object) { - return object; - }).forEach(indexItem); - - scheduleFlush(); - }); }, FLUSH_INTERVAL); worker.onmessage = handleResponse; From b632926d8e1912687df5d7693caed6f785e489fe Mon Sep 17 00:00:00 2001 From: Victor Woeltjen Date: Wed, 30 Sep 2015 13:09:55 -0700 Subject: [PATCH 12/19] [Search] Fix mutation.listen Update mutation.listen to match previous variable names changes related to nasa/openmctweb#141. --- platform/core/src/capabilities/MutationCapability.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/core/src/capabilities/MutationCapability.js b/platform/core/src/capabilities/MutationCapability.js index 8746b9af64..1ff2b6828f 100644 --- a/platform/core/src/capabilities/MutationCapability.js +++ b/platform/core/src/capabilities/MutationCapability.js @@ -171,7 +171,7 @@ define( * @memberof platform/core.MutationCapability# */ MutationCapability.prototype.listen = function (listener) { - return this.mutationTopic.listen(listener); + return this.specificMutationTopic.listen(listener); }; /** From 0d1f3bf87a90671f9a1763ff95585d24c3535c32 Mon Sep 17 00:00:00 2001 From: Victor Woeltjen Date: Wed, 16 Sep 2015 15:23:08 -0700 Subject: [PATCH 13/19] [Throttle] Update spec Conflicts: platform/features/layout/test/FixedControllerSpec.js --- platform/core/test/services/ThrottleSpec.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/platform/core/test/services/ThrottleSpec.js b/platform/core/test/services/ThrottleSpec.js index bcaf2af363..3b361f70bb 100644 --- a/platform/core/test/services/ThrottleSpec.js +++ b/platform/core/test/services/ThrottleSpec.js @@ -45,7 +45,9 @@ define( // Verify precondition: Not called at throttle-time expect(mockTimeout).not.toHaveBeenCalled(); expect(throttled()).toEqual(mockPromise); - expect(mockTimeout).toHaveBeenCalledWith(mockFn, 0, false); + expect(mockFn).not.toHaveBeenCalled(); + expect(mockTimeout) + .toHaveBeenCalledWith(jasmine.any(Function), 0, false); }); it("schedules only one timeout at a time", function () { @@ -59,10 +61,11 @@ define( it("schedules additional invocations after resolution", function () { var throttled = throttle(mockFn); throttled(); - mockPromise.then.mostRecentCall.args[0](); // Resolve timeout + mockTimeout.mostRecentCall.args[0](); // Resolve timeout throttled(); - mockPromise.then.mostRecentCall.args[0](); + mockTimeout.mostRecentCall.args[0](); throttled(); + mockTimeout.mostRecentCall.args[0](); expect(mockTimeout.calls.length).toEqual(3); }); }); From d04c5e68581f77e225006d25bd6657b866453141 Mon Sep 17 00:00:00 2001 From: Victor Woeltjen Date: Wed, 30 Sep 2015 17:08:47 -0700 Subject: [PATCH 14/19] [Search] Update GenericSearch spec nasa/openmctweb#141. --- .../services/GenericSearchProviderSpec.js | 178 ++++++++++++------ 1 file changed, 122 insertions(+), 56 deletions(-) diff --git a/platform/search/test/services/GenericSearchProviderSpec.js b/platform/search/test/services/GenericSearchProviderSpec.js index 2da7cd343b..aeeef59908 100644 --- a/platform/search/test/services/GenericSearchProviderSpec.js +++ b/platform/search/test/services/GenericSearchProviderSpec.js @@ -31,35 +31,51 @@ define( describe("The generic search provider ", function () { var mockQ, - mockTimeout, + mockLog, + mockThrottle, mockDeferred, mockObjectService, mockObjectPromise, + mockChainedPromise, mockDomainObjects, mockCapability, mockCapabilityPromise, mockWorkerService, mockWorker, + mockTopic, + mockMutationTopic, mockRoots = ['root1', 'root2'], provider, mockProviderResults; - beforeEach(function () { + function resolveObjectPromises() { var i; - + for (i = 0; i < mockObjectPromise.then.calls.length; i += 1) { + mockChainedPromise.then.calls[i].args[0]( + mockObjectPromise.then.calls[i] + .args[0](mockDomainObjects) + ); + } + } + + beforeEach(function () { mockQ = jasmine.createSpyObj( "$q", [ "defer" ] ); + mockLog = jasmine.createSpyObj( + "$log", + [ "error", "warn", "info", "debug" ] + ); mockDeferred = jasmine.createSpyObj( "deferred", [ "resolve", "reject"] ); mockDeferred.promise = "mock promise"; mockQ.defer.andReturn(mockDeferred); - - mockTimeout = jasmine.createSpy("$timeout"); - + + mockThrottle = jasmine.createSpy("throttle"); + mockObjectService = jasmine.createSpyObj( "objectService", [ "getObjects" ] @@ -68,9 +84,14 @@ define( "promise", [ "then", "catch" ] ); + mockChainedPromise = jasmine.createSpyObj( + "chainedPromise", + [ "then" ] + ); mockObjectService.getObjects.andReturn(mockObjectPromise); - - + + mockTopic = jasmine.createSpy('topic'); + mockWorkerService = jasmine.createSpyObj( "workerService", [ "run" ] @@ -80,68 +101,104 @@ define( [ "postMessage" ] ); mockWorkerService.run.andReturn(mockWorker); - + mockCapabilityPromise = jasmine.createSpyObj( "promise", [ "then", "catch" ] ); - + mockDomainObjects = {}; - for (i = 0; i < 4; i += 1) { - mockDomainObjects[i] = ( + ['a', 'root1', 'root2'].forEach(function (id) { + mockDomainObjects[id] = ( jasmine.createSpyObj( "domainObject", [ "getId", "getModel", "hasCapability", "getCapability", "useCapability" ] ) ); - mockDomainObjects[i].getId.andReturn(i); - mockDomainObjects[i].getCapability.andReturn(mockCapability); - mockDomainObjects[i].useCapability.andReturn(mockCapabilityPromise); - } - // Give the first object children - mockDomainObjects[0].hasCapability.andReturn(true); + mockDomainObjects[id].getId.andReturn(id); + mockDomainObjects[id].getCapability.andReturn(mockCapability); + mockDomainObjects[id].useCapability.andReturn(mockCapabilityPromise); + mockDomainObjects[id].getModel.andReturn({}); + }); + mockCapability = jasmine.createSpyObj( "capability", [ "invoke", "listen" ] ); mockCapability.invoke.andReturn(mockCapabilityPromise); - mockDomainObjects[0].getCapability.andReturn(mockCapability); - - provider = new GenericSearchProvider(mockQ, mockTimeout, mockObjectService, mockWorkerService, mockRoots); + mockDomainObjects.a.getCapability.andReturn(mockCapability); + mockMutationTopic = jasmine.createSpyObj( + 'mutationTopic', + [ 'listen' ] + ); + mockTopic.andCallFake(function (key) { + return key === 'mutation' && mockMutationTopic; + }); + mockThrottle.andCallFake(function (fn) { + return fn; + }); + mockObjectPromise.then.andReturn(mockChainedPromise); + + + provider = new GenericSearchProvider( + mockQ, + mockLog, + mockThrottle, + mockObjectService, + mockWorkerService, + mockTopic, + mockRoots + ); }); - + it("indexes tree on initialization", function () { + var i; + expect(mockObjectService.getObjects).toHaveBeenCalled(); expect(mockObjectPromise.then).toHaveBeenCalled(); - - // Call through the root-getting part - mockObjectPromise.then.mostRecentCall.args[0](mockDomainObjects); - - // Call through the children-getting part - mockTimeout.mostRecentCall.args[0](); - // Array argument indicates multiple children - mockCapabilityPromise.then.mostRecentCall.args[0]([]); - mockTimeout.mostRecentCall.args[0](); - // Call again, but for single child - mockCapabilityPromise.then.mostRecentCall.args[0]({}); - mockTimeout.mostRecentCall.args[0](); - - expect(mockWorker.postMessage).toHaveBeenCalled(); + + // Call through the root-getting part + resolveObjectPromises(); + + mockRoots.forEach(function (id) { + expect(mockWorker.postMessage).toHaveBeenCalledWith({ + request: 'index', + model: mockDomainObjects[id].getModel(), + id: id + }); + }); }); - - it("when indexing, listens for composition changes", function () { - var mockListener = {composition: {}}; - - // Call indexItems - mockObjectPromise.then.mostRecentCall.args[0](mockDomainObjects); - - // Call through listening for changes - expect(mockCapability.listen).toHaveBeenCalled(); - mockCapability.listen.mostRecentCall.args[0](mockListener); - expect(mockObjectService.getObjects).toHaveBeenCalled(); - mockObjectPromise.then.mostRecentCall.args[0](mockDomainObjects); + + it("indexes members of composition", function () { + mockDomainObjects.root1.getModel.andReturn({ + composition: ['a'] + }); + + resolveObjectPromises(); + resolveObjectPromises(); + + expect(mockWorker.postMessage).toHaveBeenCalledWith({ + request: 'index', + model: mockDomainObjects.a.getModel(), + id: 'a' + }); }); - + + it("listens for changes to mutation", function () { + expect(mockMutationTopic.listen) + .toHaveBeenCalledWith(jasmine.any(Function)); + mockMutationTopic.listen.mostRecentCall + .args[0](mockDomainObjects.a); + + resolveObjectPromises(); + + expect(mockWorker.postMessage).toHaveBeenCalledWith({ + request: 'index', + model: mockDomainObjects.a.getModel(), + id: mockDomainObjects.a.getId() + }); + }); + it("sends search queries to the worker", function () { var timestamp = Date.now(); provider.query(' test "query" ', timestamp, 1, 2); @@ -153,20 +210,20 @@ define( timeout: 2 }); }); - + it("gives an empty result for an empty query", function () { var timestamp = Date.now(), queryOutput; - + queryOutput = provider.query('', timestamp, 1, 2); expect(queryOutput.hits).toEqual([]); expect(queryOutput.total).toEqual(0); - + queryOutput = provider.query(); expect(queryOutput.hits).toEqual([]); expect(queryOutput.total).toEqual(0); }); - + it("handles responses from the worker", function () { var timestamp = Date.now(), event = { @@ -181,13 +238,22 @@ define( timestamp: timestamp } }; - + provider.query(' test "query" ', timestamp); mockWorker.onmessage(event); mockObjectPromise.then.mostRecentCall.args[0](mockDomainObjects); expect(mockDeferred.resolve).toHaveBeenCalled(); }); - + + it("warns when objects are unavailable", function () { + resolveObjectPromises(); + expect(mockLog.warn).not.toHaveBeenCalled(); + mockChainedPromise.then.mostRecentCall.args[0]( + mockObjectPromise.then.mostRecentCall.args[1]() + ); + expect(mockLog.warn).toHaveBeenCalled(); + }); + }); } -); \ No newline at end of file +); From bea5002752fbf46495605ff41f4f7b4756013a9f Mon Sep 17 00:00:00 2001 From: Victor Woeltjen Date: Wed, 30 Sep 2015 17:23:52 -0700 Subject: [PATCH 15/19] [Search] Add test cases Add test cases related to throttled loading of domain objects to index, nasa/openmctweb#141. --- .../services/GenericSearchProviderSpec.js | 52 +++++++++++++++---- 1 file changed, 43 insertions(+), 9 deletions(-) diff --git a/platform/search/test/services/GenericSearchProviderSpec.js b/platform/search/test/services/GenericSearchProviderSpec.js index aeeef59908..e3ee0a97ba 100644 --- a/platform/search/test/services/GenericSearchProviderSpec.js +++ b/platform/search/test/services/GenericSearchProviderSpec.js @@ -45,6 +45,8 @@ define( mockTopic, mockMutationTopic, mockRoots = ['root1', 'root2'], + mockThrottledFn, + throttledCallCount, provider, mockProviderResults; @@ -58,6 +60,18 @@ define( } } + function resolveThrottledFn() { + if (mockThrottledFn.calls.length > throttledCallCount) { + mockThrottle.mostRecentCall.args[0](); + throttledCallCount = mockThrottledFn.calls.length; + } + } + + function resolveAsyncTasks() { + resolveThrottledFn(); + resolveObjectPromises(); + } + beforeEach(function () { mockQ = jasmine.createSpyObj( "$q", @@ -75,6 +89,8 @@ define( mockQ.defer.andReturn(mockDeferred); mockThrottle = jasmine.createSpy("throttle"); + mockThrottledFn = jasmine.createSpy("throttledFn"); + throttledCallCount = 0; mockObjectService = jasmine.createSpyObj( "objectService", @@ -112,7 +128,13 @@ define( mockDomainObjects[id] = ( jasmine.createSpyObj( "domainObject", - [ "getId", "getModel", "hasCapability", "getCapability", "useCapability" ] + [ + "getId", + "getModel", + "hasCapability", + "getCapability", + "useCapability" + ] ) ); mockDomainObjects[id].getId.andReturn(id); @@ -134,12 +156,9 @@ define( mockTopic.andCallFake(function (key) { return key === 'mutation' && mockMutationTopic; }); - mockThrottle.andCallFake(function (fn) { - return fn; - }); + mockThrottle.andReturn(mockThrottledFn); mockObjectPromise.then.andReturn(mockChainedPromise); - provider = new GenericSearchProvider( mockQ, mockLog, @@ -154,6 +173,8 @@ define( it("indexes tree on initialization", function () { var i; + resolveThrottledFn(); + expect(mockObjectService.getObjects).toHaveBeenCalled(); expect(mockObjectPromise.then).toHaveBeenCalled(); @@ -174,8 +195,8 @@ define( composition: ['a'] }); - resolveObjectPromises(); - resolveObjectPromises(); + resolveAsyncTasks(); + resolveAsyncTasks(); expect(mockWorker.postMessage).toHaveBeenCalledWith({ request: 'index', @@ -190,7 +211,7 @@ define( mockMutationTopic.listen.mostRecentCall .args[0](mockDomainObjects.a); - resolveObjectPromises(); + resolveAsyncTasks(); expect(mockWorker.postMessage).toHaveBeenCalledWith({ request: 'index', @@ -246,7 +267,7 @@ define( }); it("warns when objects are unavailable", function () { - resolveObjectPromises(); + resolveAsyncTasks(); expect(mockLog.warn).not.toHaveBeenCalled(); mockChainedPromise.then.mostRecentCall.args[0]( mockObjectPromise.then.mostRecentCall.args[1]() @@ -254,6 +275,19 @@ define( expect(mockLog.warn).toHaveBeenCalled(); }); + it("throttles the loading of objects to index", function () { + expect(mockObjectService.getObjects).not.toHaveBeenCalled(); + resolveThrottledFn(); + expect(mockObjectService.getObjects).toHaveBeenCalled(); + }); + + it("logs when all objects have been processed", function () { + expect(mockLog.info).not.toHaveBeenCalled(); + resolveAsyncTasks(); + resolveThrottledFn(); + expect(mockLog.info).toHaveBeenCalled(); + }); + }); } ); From 77b0086d18e986736de7d629432531d896356537 Mon Sep 17 00:00:00 2001 From: Victor Woeltjen Date: Mon, 5 Oct 2015 09:54:57 -0700 Subject: [PATCH 16/19] [Generic Search] Use appropriate data structure Per code review, nasa/openmctweb#165 --- platform/search/src/services/GenericSearchProvider.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/platform/search/src/services/GenericSearchProvider.js b/platform/search/src/services/GenericSearchProvider.js index a8387b143f..586cc48a79 100644 --- a/platform/search/src/services/GenericSearchProvider.js +++ b/platform/search/src/services/GenericSearchProvider.js @@ -53,7 +53,7 @@ define( function GenericSearchProvider($q, $log, throttle, objectService, workerService, topic, ROOTS) { var indexed = {}, pendingQueries = {}, - toRequest = {}, + toRequest = [], worker = workerService.run('genericSearchWorker'), mutationTopic = topic("mutation"), indexingStarted = Date.now(), @@ -70,7 +70,7 @@ define( ids.forEach(function (id) { if (!indexed[id]) { indexed[id] = true; - toRequest[id] = true; + toRequest.push(id); } }); scheduleFlush(); @@ -127,7 +127,6 @@ define( } function requestAndIndex(id) { - delete toRequest[id]; pendingRequests += 1; objectService.getObjects([id]).then(function (objects) { if (objects[id]) { @@ -145,15 +144,14 @@ define( var batchSize = Math.max(MAX_CONCURRENT_REQUESTS - pendingRequests, 0); - if (Object.keys(toRequest).length + pendingRequests < 1) { + if (toRequest.length + pendingRequests < 1) { $log.info([ 'GenericSearch finished indexing after ', ((Date.now() - indexingStarted) / 1000).toFixed(2), ' seconds.' ].join('')); } else { - Object.keys(toRequest) - .slice(0, batchSize) + toRequest.splice(-batchSize, batchSize) .forEach(requestAndIndex); } }, FLUSH_INTERVAL); From 5520d90984a48acd0e329d7707bf7dbe0adbbb25 Mon Sep 17 00:00:00 2001 From: Victor Woeltjen Date: Mon, 5 Oct 2015 09:57:46 -0700 Subject: [PATCH 17/19] [Generic Search] Remove comments Per code review, nasa/openmctweb#165 --- platform/core/src/capabilities/MutationCapability.js | 2 -- platform/core/src/services/Throttle.js | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/platform/core/src/capabilities/MutationCapability.js b/platform/core/src/capabilities/MutationCapability.js index 1ff2b6828f..1f08abda6b 100644 --- a/platform/core/src/capabilities/MutationCapability.js +++ b/platform/core/src/capabilities/MutationCapability.js @@ -127,9 +127,7 @@ define( useTimestamp = arguments.length > 1; function notifyListeners(model) { - // Broadcast a general event... generalTopic.notify(domainObject); - // ...and also notify listeners watching this specific object. specificTopic.notify(model); } diff --git a/platform/core/src/services/Throttle.js b/platform/core/src/services/Throttle.js index 60444ad6c4..4b1ad32530 100644 --- a/platform/core/src/services/Throttle.js +++ b/platform/core/src/services/Throttle.js @@ -61,7 +61,7 @@ define( * @memberof platform/core.Throttle# */ return function (fn, delay, apply) { - var promise, // Promise for the result of throttled function + var promise, args = []; function invoke() { From 36d06e8b5429672fd9e3cfaf0fb461bdee5ed8b5 Mon Sep 17 00:00:00 2001 From: Victor Woeltjen Date: Mon, 5 Oct 2015 10:06:55 -0700 Subject: [PATCH 18/19] [Generic Search] Reduce flush interval Per discussion from nasa/openmctweb#141, minimize the interval at which new objects get indexed, instead of presuming requirements for CPU utilization. --- platform/search/src/services/GenericSearchProvider.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/search/src/services/GenericSearchProvider.js b/platform/search/src/services/GenericSearchProvider.js index 586cc48a79..80146f17f1 100644 --- a/platform/search/src/services/GenericSearchProvider.js +++ b/platform/search/src/services/GenericSearchProvider.js @@ -32,7 +32,7 @@ define( var DEFAULT_MAX_RESULTS = 100, DEFAULT_TIMEOUT = 1000, MAX_CONCURRENT_REQUESTS = 100, - FLUSH_INTERVAL = 180, + FLUSH_INTERVAL = 0, stopTime; /** From 134b749bbf5efcc1b1d22e29575342d6e6f32a01 Mon Sep 17 00:00:00 2001 From: Victor Woeltjen Date: Mon, 5 Oct 2015 10:11:38 -0700 Subject: [PATCH 19/19] [Generic Search] Track pending indexes Track domain objects which have indexing operations pending, to avoid redundantly indexing in cases where a broader indexed check is insufficient. --- platform/search/src/services/GenericSearchProvider.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/platform/search/src/services/GenericSearchProvider.js b/platform/search/src/services/GenericSearchProvider.js index 80146f17f1..9574d156fb 100644 --- a/platform/search/src/services/GenericSearchProvider.js +++ b/platform/search/src/services/GenericSearchProvider.js @@ -52,6 +52,7 @@ define( */ function GenericSearchProvider($q, $log, throttle, objectService, workerService, topic, ROOTS) { var indexed = {}, + pendingIndex = {}, pendingQueries = {}, toRequest = [], worker = workerService.run('genericSearchWorker'), @@ -68,8 +69,9 @@ define( function scheduleIdsForIndexing(ids) { ids.forEach(function (id) { - if (!indexed[id]) { + if (!indexed[id] && !pendingIndex[id]) { indexed[id] = true; + pendingIndex[id] = true; toRequest.push(id); } }); @@ -129,6 +131,7 @@ define( function requestAndIndex(id) { pendingRequests += 1; objectService.getObjects([id]).then(function (objects) { + delete pendingIndex[id]; if (objects[id]) { indexItem(objects[id]); }