CBG-3727: Diagnostic API: Get all doc channels (#6701)

* Add handling and test for get all doc channels

* Fix lint

* Add licensing

* Address comments

* Add docs

* fix yamllint

* Add DocumentHistoryMaxEntriesPerChannel test case and fix docs

* Denote compacted sequence pairs with ~

* Fix goimports

* Fix comment

* Fix comment

* Add unmarshalJSON handlign for compacted sequence pair

auth/role.go

@@ -59,13 +59,19 @@ type GrantHistory struct {
 // Struct is for ease of internal use
 // Bucket store has each entry as a string "seq-endSeq"
 type GrantHistorySequencePair struct {
-	StartSeq uint64 // Sequence at which a grant was performed to give access to a role / channel. Only populated once endSeq is available.
-	EndSeq   uint64 // Sequence when access to a role / channel was revoked.
+	StartSeq  uint64 // Sequence at which a grant was performed to give access to a role / channel. Only populated once endSeq is available.
+	EndSeq    uint64 // Sequence when access to a role / channel was revoked.
+	Compacted bool
 }
 
-// MarshalJSON will handle conversion from having a seq / endSeq struct to the bucket format of "seq-endSeq"
+// MarshalJSON will handle conversion from having a seq / endSeq struct to the bucket format of "seq-endSeq" and "seq~endSeq" if the pair was compacted
 func (pair *GrantHistorySequencePair) MarshalJSON() ([]byte, error) {
-	stringPair := fmt.Sprintf("%d-%d", pair.StartSeq, pair.EndSeq)
+	var stringPair string
+	if pair.Compacted {
+		stringPair = fmt.Sprintf("%d~%d", pair.StartSeq, pair.EndSeq)
+	} else {
+		stringPair = fmt.Sprintf("%d-%d", pair.StartSeq, pair.EndSeq)
+	}
 	return base.JSONMarshal(stringPair)
 }
 
@@ -80,7 +86,12 @@ func (pair *GrantHistorySequencePair) UnmarshalJSON(data []byte) error {
 
 	splitPair := strings.Split(stringPair, "-")
 	if len(splitPair) != 2 {
-		return fmt.Errorf("unexpected sequence pair length")
+		// try again with compacted sequence pair format
+		splitPair = strings.Split(stringPair, "~")
+		if len(splitPair) != 2 {
+			return fmt.Errorf("unexpected sequence pair length")
+		}
+		pair.Compacted = true
 	}
 
 	pair.StartSeq, err = strconv.ParseUint(splitPair[0], 10, 64)

db/document.go

@@ -57,9 +57,10 @@ type UserAccessMap map[string]channels.TimedSet
 type AttachmentsMeta map[string]interface{} // AttachmentsMeta metadata as included in sync metadata
 
 type ChannelSetEntry struct {
-	Name  string `json:"name"`
-	Start uint64 `json:"start"`
-	End   uint64 `json:"end,omitempty"`
+	Name      string `json:"name"`
+	Start     uint64 `json:"start"`
+	End       uint64 `json:"end,omitempty"`
+	Compacted bool   `json:"compacted,omitempty"`
 }
 
 // The sync-gateway metadata stored in the "_sync" property of a Couchbase document.
@@ -930,6 +931,7 @@ func (doc *Document) addToChannelSetHistory(channelName string, historyEntry Cha
 
 	if entryCount >= DocumentHistoryMaxEntriesPerChannel {
 		doc.ChannelSetHistory[secondOldestEntryIdx].Start = oldestEntryStartSeq
+		doc.ChannelSetHistory[secondOldestEntryIdx].Compacted = true
 		doc.ChannelSetHistory = append(doc.ChannelSetHistory[:oldestEntryIdx], doc.ChannelSetHistory[oldestEntryIdx+1:]...)
 	}
 

docs/api/diagnostic.yaml

@@ -30,6 +30,8 @@ servers:
 paths:
   /_ping:
     $ref: ./paths/common/_ping.yaml
+  '/{keyspace}/{docid}/_all_channels':
+    $ref: './paths/diagnostic/keyspace-docid-_all_channels.yaml'
 externalDocs:
   description: Sync Gateway Quickstart | Couchbase Docs
   url: 'https://docs.couchbase.com/sync-gateway/current/index.html'

docs/api/paths/diagnostic/keyspace-docid-_all_channels.yaml

@@ -0,0 +1,37 @@
+# Copyright 2022-Present Couchbase, Inc.
+#
+# Use of this software is governed by the Business Source License included
+# in the file licenses/BSL-Couchbase.txt.  As of the Change Date specified
+# in that file, in accordance with the Business Source License, use of this
+# software will be governed by the Apache License, Version 2.0, included in
+# the file licenses/APL2.txt.
+parameters:
+  - $ref: ../../components/parameters.yaml#/keyspace
+  - $ref: ../../components/parameters.yaml#/docid
+get:
+  summary: Get channel history for a document
+  description: |-
+    Retrieve all doc channels and the sequence spans showing when the doc was added to a channel and when it was removed.
+    Required Sync Gateway RBAC roles:
+    * Sync Gateway Application Read Only
+  responses:
+    '200':
+      description: Document found successfully
+      content:
+        application/json:
+          schema:
+            additionalProperties:
+              x-additionalPropertiesName: channel
+              description: The channels the document has been in.
+              type: array
+              items:
+                sequences:
+                description: The sequence number that document was added to the channel.
+                type: string
+                example: "28-48"
+
+    '404':
+      $ref: ../../components/responses.yaml#/Not-found
+  tags:
+    - Document
+  operationId: get_keyspace-docid-_all_channels

rest/diagnostic_doc_api.go

@@ -0,0 +1,41 @@
+/*
+Copyright 2024-Present Couchbase, Inc.
+
+Use of this software is governed by the Business Source License included in
+the file licenses/BSL-Couchbase.txt.  As of the Change Date specified in that
+file, in accordance with the Business Source License, use of this software will
+be governed by the Apache License, Version 2.0, included in the file
+licenses/APL2.txt.
+*/
+
+package rest
+
+import (
+	"github.com/couchbase/sync_gateway/auth"
+	"github.com/couchbase/sync_gateway/db"
+)
+
+// HTTP handler for a GET of a document
+func (h *handler) handleGetDocChannels() error {
+	docid := h.PathVar("docid")
+
+	doc, err := h.collection.GetDocument(h.ctx(), docid, db.DocUnmarshalSync)
+	if err != nil {
+		return err
+	}
+	if doc == nil {
+		return kNotFoundError
+	}
+	resp := make(map[string][]auth.GrantHistorySequencePair, len(doc.Channels))
+
+	for _, chanSetInfo := range doc.SyncData.ChannelSet {
+		resp[chanSetInfo.Name] = append(resp[chanSetInfo.Name], auth.GrantHistorySequencePair{StartSeq: chanSetInfo.Start, EndSeq: chanSetInfo.End})
+	}
+	for _, hist := range doc.SyncData.ChannelSetHistory {
+		resp[hist.Name] = append(resp[hist.Name], auth.GrantHistorySequencePair{StartSeq: hist.Start, EndSeq: hist.End, Compacted: hist.Compacted})
+		continue
+	}
+
+	h.writeJSON(resp)
+	return nil
+}

rest/diagnostic_doc_api_test.go

@@ -0,0 +1,59 @@
+/*
+Copyright 2024-Present Couchbase, Inc.
+
+Use of this software is governed by the Business Source License included in
+the file licenses/BSL-Couchbase.txt.  As of the Change Date specified in that
+file, in accordance with the Business Source License, use of this software will
+be governed by the Apache License, Version 2.0, included in the file
+licenses/APL2.txt.
+*/
+
+package rest
+
+import (
+	"encoding/json"
+	"net/http"
+	"testing"
+
+	"github.com/couchbase/sync_gateway/db"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetAlldocChannels(t *testing.T) {
+	rt := NewRestTester(t, &RestTesterConfig{SyncFn: `function(doc) {channel(doc.channel);}`})
+	defer rt.Close()
+
+	version := rt.PutDoc("doc", `{"channel":["CHAN1"]}`)
+	updatedVersion := rt.UpdateDoc("doc", version, `{"channel":["CHAN2"]}`)
+	updatedVersion = rt.UpdateDoc("doc", updatedVersion, `{"channel":["CHAN1"]}`)
+	updatedVersion = rt.UpdateDoc("doc", updatedVersion, `{"channel":["CHAN1", "CHAN2"]}`)
+	updatedVersion = rt.UpdateDoc("doc", updatedVersion, `{"channel":["CHAN3"]}`)
+	updatedVersion = rt.UpdateDoc("doc", updatedVersion, `{"channel":["CHAN1"]}`)
+
+	response := rt.SendDiagnosticRequest("GET", "/{{.keyspace}}/doc/_all_channels", "")
+	RequireStatus(t, response, http.StatusOK)
+
+	var channelMap map[string][]string
+	err := json.Unmarshal(response.BodyBytes(), &channelMap)
+	assert.NoError(t, err)
+	assert.ElementsMatch(t, channelMap["CHAN1"], []string{"6-0", "1-2", "3-5"})
+	assert.ElementsMatch(t, channelMap["CHAN2"], []string{"4-5", "2-3"})
+	assert.ElementsMatch(t, channelMap["CHAN3"], []string{"5-6"})
+
+	for i := 1; i <= 10; i++ {
+		updatedVersion = rt.UpdateDoc("doc", updatedVersion, `{}`)
+		updatedVersion = rt.UpdateDoc("doc", updatedVersion, `{"channel":["CHAN3"]}`)
+	}
+	response = rt.SendAdminRequest("GET", "/{{.keyspace}}/doc", "")
+	RequireStatus(t, response, http.StatusOK)
+	response = rt.SendDiagnosticRequest("GET", "/{{.keyspace}}/doc/_all_channels", "")
+	RequireStatus(t, response, http.StatusOK)
+
+	err = json.Unmarshal(response.BodyBytes(), &channelMap)
+	assert.NoError(t, err)
+
+	// If the channel is still in channel_set, then the total will be 5 entries in history and 1 in channel_set
+	assert.Equal(t, len(channelMap["CHAN3"]), db.DocumentHistoryMaxEntriesPerChannel+1)
+
+}

rest/routing.go

@@ -368,7 +368,11 @@ func CreateMetricRouter(sc *ServerContext) *mux.Router {
 
 func createDiagnosticRouter(sc *ServerContext) *mux.Router {
 	r := CreatePingRouter(sc)
-
+	dbr := r.PathPrefix("/{db:" + dbRegex + "}/").Subrouter()
+	dbr.StrictSlash(true)
+	keyspace := r.PathPrefix("/{keyspace:" + dbRegex + "}/").Subrouter()
+	keyspace.StrictSlash(true)
+	keyspace.Handle("/{docid:"+docRegex+"}/_all_channels", makeHandler(sc, adminPrivs, []Permission{PermReadAppData}, nil, (*handler).handleGetDocChannels)).Methods("GET")
 	return r
 }