[#65] Backend logic to handle triggers and global queries, plus fix s…

…ome old tests that had become broken

src/server/models/TableModel.gs

@@ -7,30 +7,27 @@ var defaultTableConfig_ = {
 //        "timed": false,
 //        "refresh_s": 60
 //     },
+     "global_triggers": [], //array of ranges ("sheet!range") to include when deciding whether to refresh the table
      "query": {
 //       "index_pattern": "tbd",
        "source": "none",
-        //, //points to field to use ("global", "local", "fixed") .. NOT_SUPPORTED: "global", "fixed"
-//        "global": {
-//           "range_name": "tbd"
-//        },
+//,       //^points to field to use ("global", "local")
+       "global": {
+          "range_name": "sheet!range"
+       },
         "local": {
-           "position": "top" //(or "bottom" ... NOT_SUPPORTED: "bottom")
+           "position": "top" //(or "bottom")
         }
-        //,
-//        "fixed": {
-//           "string": "{} or SQL or lucene"
-//        }
      },
      "pagination": {
         "source": "none",
-        //, //points to field to use ("global", "local", "fixed") .. NOT_SUPPORTED: "global", "fixed"
+        //, //points to field to use ("global", "local") .. NOT_SUPPORTED: "global"
 //       "global": {
 //          "enabled": false,
 //          "range_name": "tbd"
 //       },
        "local": {
-          "position": "bottom" //(or "top") .. NOT_SUPPORTED: "top"
+          "position": "bottom" //(or "top")
        }
      },
      "status": {
@@ -43,12 +40,12 @@ var defaultTableConfig_ = {
           "# eg -x.*.y / +x.y (start with -s where possible)",
           "# pre-built groups: $$<name>",
           "#(note fields are laid out in match order)"
-        ], //(# to ignore an entry, [+-] to be +ve/-ve selection, // for regex else */** for single/multi path wildcard)
+        ], //(# to ignore an entry, [+-] to be +ve/-ve selection, // for regex else * for full wildcard)
         "exclude_filtered_fields_from_autocomplete": true,
         "autocomplete_filters": [ //(only affects autocomplete - eg for aggregations)
           "# eg x, -x.*.y, +x.y (start with -s if possible)",
           "# pre-built groups: $$<name>"
-        ], //(# to ignore an entry, [+-] to be +ve/-ve selection, // for regex else */** for single/multi path wildcard)
+        ], //(# to ignore an entry, [+-] to be +ve/-ve selection, // for regex else * for full wildcard)
         "field_aliases": [
           "#field.path=Alias To Use",
           "#(note fields are laid out in order within filter matches)"
@@ -58,9 +55,9 @@ var defaultTableConfig_ = {
         "include_note": true, //(adds a note to the top left of each table with the name)
         "theme": "minimal" //(or "none", in the future: "default", etc)
      },
-     "skip": {
-        "rows": "", //comma-separated list of offsets
-        "cols": "", //comma-separated list of offsets
+     "skip": { //NOT_SUPPORTED
+//        "rows": "", //comma-separated list of offsets
+//        "cols": "", //comma-separated list of offsets
      }
      //,
 //     "rotated": false, //(left-to-right instead of top-to-bottom)
@@ -86,7 +83,7 @@ var defaultTableConfig_ = {
       "size": "$$pagination_size",
       "_source": "$$field_filters"
     },
-    "script_fields": [] 
+    "script_fields": []
     // format is {
     //  name: "string", // the name used in the output column
     //  source: "string", // the script itself

src/server/services/ElasticsearchService.gs

@@ -287,6 +287,7 @@ var ElasticsearchService_ = (function() {
 
   /** Trigger for edit */
   function handleContentUpdates(event, triggerOverride) {
+    var ss = SpreadsheetApp.getActive()
     //(copy paste from ElasticsearchManager.isTriggerEnabled_)
     var isTriggerEnabled = function(tableConfig, trigger) {
       var tableTrigger = tableConfig.trigger || "control_change"
@@ -316,6 +317,8 @@ var ElasticsearchService_ = (function() {
       var tableConfig = matchingTables[matchingTableName]
       var activeRange = tableConfig.activeRange
       delete tableConfig.activeRange //(remove extra non-standard field)
+      delete tableConfig.temp //(this is a copy of the meta so doesn't do anything)
+        //TODO: ^ really we should use the last table config that was _tested_?
       tableConfig = tableConfig.temp ? tableConfig.temp : tableConfig //(use current version, not saved)
 
       // Logic to determine if a table edit hits the control cells (query/page)
@@ -336,6 +339,16 @@ var ElasticsearchService_ = (function() {
             return TableRangeUtils_.doRangesIntersect(event.range, newRange)
           })
 
+        // Also handle any global triggers (including queries):
+        var globalTriggerRanges = TableRangeUtils_.getExternalTableRanges(ss, tableConfig)
+        modifiedOffsets = modifiedOffsets.concat(
+          globalTriggerRanges.filter(function(triggerRange) {
+            return TableRangeUtils_.doRangesIntersect(event.range, triggerRange)
+          }).map(function(rangeNotation) {
+            return "query_offset" //(so that will be treated like a control change)
+          })
+        )
+
         if (modifiedOffsets.length > 0) {
           if (modifiedOffsets.indexOf("query_offset") >= 0) { //query has changed...
             //...if the page is hand specified, reset to 1

src/server/services/TableService.gs

@@ -173,11 +173,18 @@ var TableService_ = (function(){
     var namedRangeMap = TableRangeUtils_.listTableRanges(ss, Object.keys(tableMap))
     var retVal = {}
     Object.keys(namedRangeMap).forEach(function(tableName) {
+      var tableConfig = tableMap[tableName]
+      var globalTriggerRanges = TableRangeUtils_.getExternalTableRanges(ss, tableConfig)
       var namedRange = namedRangeMap[tableName]
-      if (TableRangeUtils_.doRangesIntersect(range, namedRange.getRange())) {
-        retVal[tableName] = TableRangeUtils_.shallowCopy(tableMap[tableName])
-        if (addRange) {
-          retVal[tableName].activeRange = namedRange.getRange()
+      var allRangesToTest = globalTriggerRanges.concat([ namedRange.getRange() ])
+      for (rangeToTestIndex in allRangesToTest) {
+        var rangeToTest = allRangesToTest[rangeToTestIndex]
+        if (TableRangeUtils_.doRangesIntersect(range, rangeToTest)) {
+          retVal[tableName] = TableRangeUtils_.shallowCopy(tableConfig)
+          if (addRange) {
+            retVal[tableName].activeRange = namedRange.getRange()
+          }
+          break //(no more checks needed, match gets qualified in calling method)
         }
       }
     })

src/server/utils/ElasticsearchRequestUtils.gs

@@ -122,6 +122,18 @@ var ElasticsearchRequestUtils_ = (function() {
               break
         }
         retVal.query_offset = { row: queryRow, col: 2 }
+     } else if ( //Global query, get from its external location:
+       "global" == TableRangeUtils_.getJson(tableConfig, [ "common", "query", "source" ])
+     ) {
+       var ss = SpreadsheetApp.getActive()
+       var globalQueryRef = TableRangeUtils_.getJson(
+         tableConfig, [ "common", "query", "global", "range_name" ]
+       )
+       var globalQueryRange = globalQueryRef ?
+         TableRangeUtils_.getRangeFromName(ss, globalQueryRef) : null
+       if (globalQueryRange) {
+         retVal.query = globalQueryRange.getCell(1, 1).getValue()
+       }
      }
 
      // Status (if not merged)

src/server/utils/ElasticsearchResponseUtils.gs

@@ -195,7 +195,7 @@ var ElasticsearchResponseUtils_ = (function() {
   if (debugMode && debugModeSwallowException) debug.push("ERROR: [" + arrayFieldChain + "] vs [" + mutableState.bottom_path + "], "); else
               throw new Error(
                     "By policy, only allowed a single chain of nested aggregations - [" + arrayFieldChain + "] vs [" + mutableState.bottom_path + "], " +
-                    "if you need intermediate buckets you can filter them out by setting 'filter_field' to '-' or '-**'"
+                    "if you need intermediate buckets you can filter them out by setting 'filter_field' to '-' or '-*'"
               )
            } else if (!mutableState.bottom_path) {
               mutableState.bottom_path = arrayFieldChain

src/server/utils/TableRangeUtils.gs

@@ -354,6 +354,40 @@
       return retVal
    }
 
+   /** For a given table, returns a list of ranges that the table monitors
+    *  and treats like control changes
+    */
+   function getExternalTableRanges(ss, tableConfig) {
+     var globalTriggers = TableRangeUtils_.getJson(tableConfig, [ "common", "global_triggers" ]) || []
+     var isGlobalQuery = "global" == TableRangeUtils_.getJson(tableConfig, [ "common", "query", "source" ])
+     var globalQuery = TableRangeUtils_.getJson(tableConfig, [ "common", "query", "global", "range_name" ])
+     if (isGlobalQuery) {
+       globalTriggers = globalTriggers.concat([ globalQuery ])
+     }
+     return globalTriggers.map(function(rangeOrNotation) {
+       return getRangeFromName(ss, rangeOrNotation)
+     }).filter(function(range) {
+       return range != null
+     })
+   }
+
+   /** Returns the range from a range notation
+    * TODO: needs to be able to handle named ranges (see getJsonLookup)
+   */
+   function getRangeFromName(ss, rangeOrNotation) {
+     var isNotation =
+      rangeOrNotation.indexOf(":") >= 0 || /^[A-Z]+[0-9]+$/.exec(rangeOrNotation)
+     var isFullNotation = rangeOrNotation.indexOf("!") >= 0
+
+     return isNotation ?
+       (isFullNotation ?
+         ss.getRange(rangeOrNotation)
+         :
+         ss.getActiveSheet().getRange(rangeOrNotation)
+       )
+       : null //(see above: also support named ranges, cf getJsonLookup)
+   }
+
    ////////////////////////////////////////////////////////
 
    // 3] Internal utils
@@ -402,6 +436,8 @@
      formatDate: formatDate,
      fixSelectAllRanges: fixSelectAllRanges,
      doRangesIntersect: doRangesIntersect,
+     getExternalTableRanges: getExternalTableRanges,
+     getRangeFromName: getRangeFromName,
 
      TESTONLY: {
 

test/server/services/TestElasticsearchService.gs

@@ -105,6 +105,9 @@
             "local": {
                "position": "none"
             },
+            "global": {
+              "range_name": "TBD"
+            },
             "source": "local"
          },
          "pagination": {
@@ -126,6 +129,7 @@
       }}
 
       var defaultA1Notation = "A1:E10"
+      var externalQueryNotation = "H20:H20"
 
       TestService_.Utils.performTest(testResults, "no_special_rows_plus_check_es_meta", function() {
          var tableConfig = TestService_.Utils.deepCopyJson(baseTableConfig)
@@ -212,7 +216,8 @@
         "query_status_pagination": { status: "top", merge: true, noteMode: 2 },
         "pagination_status": { status: "bottom", merge: true, noteMode: 3 },
         "pagination_status_test": { status: "bottom", merge: true, testMode: true, noteMode: 3 },
-        "query_status_pagination_nomerge": { status: "bottom", merge: false, noteMode: 4 }
+        "query_status_pagination_nomerge": { status: "bottom", merge: false, noteMode: 4 },
+        "global_query": { noteMode: 4 }
       }
 
       var testRunner = function(testName, testConfig) { TestService_.Utils.performTest(testResults, testName, function() {
@@ -223,7 +228,8 @@
 
          var testMode = testConfig.testMode
 
-         var includeQueryBar = testName.indexOf("query") >= 0
+         var globalQuery = testName.indexOf("global_query") >= 0
+         var includeQueryBar = !globalQuery && testName.indexOf("query") >= 0
          var includePagination = testName.indexOf("pagination") >= 0
          var includeStatus = testName.indexOf("status") >= 0
 
@@ -241,7 +247,10 @@
            default: return null
          }}())
          var expectedNote = (function() { switch(testConfig.noteMode) {
-           case 1: return testMode ? "" : "ES Table: use_named_range\n"
+//TODO: notes not supported by this test currently, they are explicitly disabled for unsaved tables
+//           case 1: return testMode ? "" : "ES Table: use_named_range\n"
+// (for the other cases we were just checking that it didn't overwrite them anyway, so can leave)
+           case 1: return ""
            case 2: return "ES Table: use_named_range\nPlus user notes"
            case 3: return testMode ? "ES Table: use_named_range\n" : ""
            case 4: return "User added note"
@@ -253,6 +262,12 @@
          var testCaseInfoArray = []
          var queryPosition = { col: 2 , row: 1 }
          tableConfig.common.formatting.include_note = includeNote
+         if (globalQuery) {
+           numTestCases *= Object.keys(queryTestCases).length
+           testCaseInfoArray.push(queryTestCases)
+           tableConfig.common.query.global.range_name = externalQueryNotation
+           tableConfig.common.query.source = "global"
+         }
          if (includeQueryBar) {
             expectedDataSize--
             numTestCases *= Object.keys(queryTestCases).length
@@ -295,6 +310,9 @@
          for (var testCaseKey in testCases) {
            var testCaseConfig = testCases[testCaseKey]
            range.clear()
+           if (globalQuery) {
+             testSheet.getRange(externalQueryNotation).setValue(testCaseConfig.query)
+           }
            if (includeQueryBar) {
               // query always set - default means "", so write to the right spot
               range.getCell(1, 1).setValue("Query:")
@@ -497,11 +515,11 @@
      var resetTables = function() {
        testSheet.getRange("B2").setValue("test status")
        testSheet.getRange("B5").setValue(2)
-       ManagementService_.updateTempSavedObject("testa", "testa", null)
+       ManagementService_.updateSavedObject("testa", TableRangeUtils_.shallowCopy(baseTableConfig))
        ManagementService_.setSavedObjectTrigger("testa", "")
        testSheet.getRange("G2").setValue("test status")
        testSheet.getRange("G5").setValue(2)
-       ManagementService_.updateTempSavedObject("testb", "testb", null)
+       ManagementService_.updateSavedObject("testb", TableRangeUtils_.shallowCopy(baseTableConfig))
        ManagementService_.setSavedObjectTrigger("testb", "")
      }
 
@@ -524,6 +542,12 @@
      var contentConfig = TestService_.Utils.deepCopyJson(baseTableConfig)
      contentConfig.trigger = "content_change"
 
+     var triggerConfig = TestService_.Utils.deepCopyJson(baseTableConfig)
+     triggerConfig.common.global_triggers = [ "B25:C25" ]
+     triggerConfig.common.query.source = "global"
+     triggerConfig.common.query.global.range_name = "A25"
+     triggerConfig.trigger = "control_change"
+
      // Now check triggers:
 
     // page of table 1
@@ -540,7 +564,7 @@
 
        //(Add a temp object with pagination turned off)
        resetTables()
-       ManagementService_.updateTempSavedObject("testa", "testa", noPagination)
+       ManagementService_.updateSavedObject("testa", TableRangeUtils_.shallowCopy(noPagination))
 
        var retVal = ElasticsearchService_.handleContentUpdates(changeEvent, /*triggerOverride*/null)
        TestService_.Utils.assertEquals(0, retVal, "retval (nopage)")
@@ -551,7 +575,7 @@
 
        // Nothing should happen if it's disabled
        resetTables()
-       ManagementService_.updateTempSavedObject("testa", "testa", disabledConfig)
+       ManagementService_.updateSavedObject("testa", TableRangeUtils_.shallowCopy(disabledConfig))
 
        var retVal = ElasticsearchService_.handleContentUpdates(changeEvent, /*triggerOverride*/null)
        TestService_.Utils.assertEquals(0, retVal, "retval (nopage)")
@@ -571,7 +595,7 @@
        tests.forEach(function(testConfig) {
          resetTables()
          if (testConfig.temp) {
-           ManagementService_.updateTempSavedObject("testb", "testb", noPagination)
+           ManagementService_.updateSavedObject("testb", TableRangeUtils_.shallowCopy(noPagination))
          }
          if (testConfig.formula) {
            testSheet.getRange("G5").setFormula("=A10")
@@ -595,7 +619,7 @@
        TestService_.Utils.performTest(testResults, "Content intersection vs [" + contentEnabled + "]", function() {
          resetTables()
          if (contentEnabled) {
-           ManagementService_.updateTempSavedObject("testb", "testb", contentConfig)
+           ManagementService_.updateSavedObject("testb", TableRangeUtils_.shallowCopy(contentConfig))
          }
          var changeEvent = { range: testSheet.getRange("A3:Z3") }
          var retVal = ElasticsearchService_.handleContentUpdates(changeEvent, /*triggerOverride*/null)
@@ -619,6 +643,36 @@
          }
        })
      })
+
+     TestService_.Utils.performTest(testResults, "Global trigger", function() {
+       // Global query trigger
+       resetTables()
+       ManagementService_.updateSavedObject("testb", TableRangeUtils_.shallowCopy(triggerConfig))
+       testSheet.getRange("G1").setValue("test status") //(no query bar)
+       var changeEvent = { range: testSheet.getRange("A22:A26") }
+       var retVal = ElasticsearchService_.handleContentUpdates(changeEvent, /*triggerOverride*/null)
+       TestService_.Utils.assertEquals(
+         1, retVal, "check handleContentUpdates return value (triggerConfig, test 1)"
+       )
+       var resultsB = getResults("testb", "F1:J5", "G1", "G5")
+       TestService_.Utils.assertEquals(
+         { p: 1, s: "AWAITING REFRESH", t: "control_change"}, resultsB[0], "tableB1=[" + resultsB[1] + "]"
+       )
+
+       // Other global triggers
+       resetTables()
+       ManagementService_.updateSavedObject("testb", TableRangeUtils_.shallowCopy(triggerConfig))
+       testSheet.getRange("G1").setValue("test status") //(no query bar)
+       var changeEvent = { range: testSheet.getRange("B22:B26") }
+       var retVal = ElasticsearchService_.handleContentUpdates(changeEvent, /*triggerOverride*/null)
+       TestService_.Utils.assertEquals(
+         1, retVal, "check handleContentUpdates return value (triggerConfig, test 2)"
+       )
+       var resultsB = getResults("testb", "F1:J5", "G1", "G5")
+       TestService_.Utils.assertEquals(
+         { p: 1, s: "AWAITING REFRESH", t: "control_change"}, resultsB[0], "tableB2=[" + resultsB[1] + "]"
+       )
+     })
    }
 
    ////////////////////////////////////////////////////////

test/server/utils/TestElasticsearchResponseUtils.gs

@@ -166,7 +166,7 @@ var TestElasticsearchResponseUtils_ = (function() {
       }
       var mr1 = { "name": "mr1" }
       if (!config.mr_enabled) {
-        mr1.field_filter = "-**"
+        mr1.field_filter = "-*"
       }
       var m2 = { "name": "m2", "field_filter": "-stat2.filter_out" }
       if (config.alt_filter) {