CBG-3721: [Diagnostic API] Import filter and sync function dry run endpoints

base/log_keys.go

@@ -41,6 +41,7 @@ const (
 	KeyConfig
 	KeyCRUD
 	KeyDCP
+	KeyDiagnostic
 	KeyEvents
 	KeyGoCB
 	KeyHTTP
@@ -74,6 +75,7 @@ var (
 		KeyConfig:         "Config",
 		KeyCRUD:           "CRUD",
 		KeyDCP:            "DCP",
+		KeyDiagnostic:     "Diagnostic",
 		KeyEvents:         "Events",
 		KeyGoCB:           "gocb",
 		KeyHTTP:           "HTTP",

db/crud.go

@@ -1145,6 +1145,89 @@ func (db *DatabaseCollectionWithUser) PutExistingRevWithBody(ctx context.Context
 
 }
 
+// SyncFnDryrun Runs a document through the sync function and returns expiry, channels doc was placed in, access map for users, roles, handler errors and sync fn exceptions
+func (db *DatabaseCollectionWithUser) SyncFnDryrun(ctx context.Context, body Body, docID string) (*channels.ChannelMapperOutput, error, error) {
+	doc := &Document{
+		ID:    docID,
+		_body: body,
+	}
+	oldDoc := doc
+	if docID != "" {
+		if docInBucket, err := db.GetDocument(ctx, docID, DocUnmarshalAll); err == nil {
+			oldDoc = docInBucket
+			if doc._body == nil {
+				body = oldDoc.Body(ctx)
+				doc._body = body
+				// If no body is given, use doc in bucket as doc with no old doc
+				oldDoc._body = nil
+			}
+			doc._body[BodyRev] = oldDoc.SyncData.CurrentRev
+		} else {
+			return nil, err, nil
+		}
+	} else {
+		oldDoc._body = nil
+	}
+
+	delete(body, BodyId)
+
+	// Get the revision ID to match, and the new generation number:
+	matchRev, _ := body[BodyRev].(string)
+	generation, _ := ParseRevID(ctx, matchRev)
+	if generation < 0 {
+		return nil, base.HTTPErrorf(http.StatusBadRequest, "Invalid revision ID"), nil
+	}
+	generation++
+
+	// Create newDoc which will be used to pass around Body
+	newDoc := &Document{
+		ID: docID,
+	}
+	// Pull out attachments
+	newDoc.DocAttachments = GetBodyAttachments(body)
+	delete(body, BodyAttachments)
+
+	delete(body, BodyRevisions)
+
+	err := validateAPIDocUpdate(body)
+	if err != nil {
+		return nil, err, nil
+	}
+	bodyWithoutInternalProps, wasStripped := stripInternalProperties(body)
+	canonicalBytesForRevID, err := base.JSONMarshalCanonical(bodyWithoutInternalProps)
+	if err != nil {
+		return nil, err, nil
+	}
+
+	// We needed to keep _deleted around in the body until we generated a rev ID, but now we can ditch it.
+	_, isDeleted := body[BodyDeleted]
+	if isDeleted {
+		delete(body, BodyDeleted)
+	}
+
+	// and now we can finally update the newDoc body to be without any special properties
+	newDoc.UpdateBody(body)
+
+	// If no special properties were stripped and document wasn't deleted, the canonical bytes represent the current
+	// body.  In this scenario, store canonical bytes as newDoc._rawBody
+	if !wasStripped && !isDeleted {
+		newDoc._rawBody = canonicalBytesForRevID
+	}
+
+	newRev := CreateRevIDWithBytes(generation, matchRev, canonicalBytesForRevID)
+	newDoc.RevID = newRev
+	mutableBody, metaMap, _, err := db.prepareSyncFn(oldDoc, newDoc)
+	if err != nil {
+		base.InfofCtx(ctx, base.KeyDiagnostic, "Failed to prepare to run sync function: %v", err)
+		return nil, err, nil
+	}
+
+	output, err := db.ChannelMapper.MapToChannelsAndAccess(ctx, mutableBody, string(oldDoc._rawBody), metaMap,
+		MakeUserCtx(db.user, db.ScopeName, db.Name))
+
+	return output, nil, err
+}
+
 // resolveConflict runs the conflictResolverFunction with doc and newDoc.  doc and newDoc's bodies and revision trees
 // may be changed based on the outcome of conflict resolution - see resolveDocLocalWins, resolveDocRemoteWins and
 // resolveDocMerge for specifics on what is changed under each scenario.
@@ -2123,7 +2206,6 @@ func getAttachmentIDsForLeafRevisions(ctx context.Context, db *DatabaseCollectio
 
 	return leafAttachments, nil
 }
-
 func (db *DatabaseCollectionWithUser) checkDocChannelsAndGrantsLimits(ctx context.Context, docID string, channels base.Set, accessGrants channels.AccessMap, roleGrants channels.AccessMap) {
 	if db.unsupportedOptions() == nil || db.unsupportedOptions().WarningThresholds == nil {
 		return

db/database_collection.go

@@ -165,7 +165,7 @@ func (c *DatabaseCollection) ForceAPIForbiddenErrors() bool {
 	return c.dbCtx.Options.UnsupportedOptions != nil && c.dbCtx.Options.UnsupportedOptions.ForceAPIForbiddenErrors
 }
 
-// importFilter returns the sync function.
+// ImportFilter returns the import filter.
 func (c *DatabaseCollection) importFilter() *ImportFilterFunction {
 	return c.importFilterFunction
 }

db/import.go

@@ -243,13 +243,13 @@ func (db *DatabaseCollectionWithUser) importDoc(ctx context.Context, docid strin
 
 			if isDelete && body == nil {
 				deleteBody := Body{BodyDeleted: true}
-				shouldImport, err = importFilter.EvaluateFunction(ctx, deleteBody)
+				shouldImport, err = importFilter.EvaluateFunction(ctx, deleteBody, false)
 			} else if isDelete && body != nil {
 				deleteBody := body.ShallowCopy()
 				deleteBody[BodyDeleted] = true
-				shouldImport, err = importFilter.EvaluateFunction(ctx, deleteBody)
+				shouldImport, err = importFilter.EvaluateFunction(ctx, deleteBody, false)
 			} else {
-				shouldImport, err = importFilter.EvaluateFunction(ctx, body)
+				shouldImport, err = importFilter.EvaluateFunction(ctx, body, false)
 			}
 
 			if err != nil {
@@ -495,11 +495,13 @@ func NewImportFilterFunction(ctx context.Context, fnSource string, timeout time.
 }
 
 // Calls a jsEventFunction returning an interface{}
-func (i *ImportFilterFunction) EvaluateFunction(ctx context.Context, doc Body) (bool, error) {
+func (i *ImportFilterFunction) EvaluateFunction(ctx context.Context, doc Body, dryRun bool) (bool, error) {
 
 	result, err := i.Call(ctx, doc)
 	if err != nil {
-		base.WarnfCtx(ctx, "Unexpected error invoking import filter for document %s - processing aborted, document will not be imported.  Error: %v", base.UD(doc), err)
+		if !dryRun {
+			base.WarnfCtx(ctx, "Unexpected error invoking import filter for document %s - processing aborted, document will not be imported.  Error: %v", base.UD(doc), err)
+		}
 		return false, err
 	}
 	switch result := result.(type) {
@@ -512,7 +514,26 @@ func (i *ImportFilterFunction) EvaluateFunction(ctx context.Context, doc Body) (
 		}
 		return boolResult, nil
 	default:
-		base.WarnfCtx(ctx, "Import filter function returned non-boolean result %v Type: %T", result, result)
+		if !dryRun {
+			base.WarnfCtx(ctx, "Import filter function returned non-boolean result %v Type: %T", result, result)
+		}
 		return false, errors.New("Import filter function returned non-boolean value.")
 	}
 }
+func (db *DatabaseCollectionWithUser) ImportFilterDryRun(ctx context.Context, doc Body, docid string) (bool, error) {
+
+	importFilter := db.importFilter()
+	if docid != "" {
+		docInBucket, err := db.GetDocument(ctx, docid, DocUnmarshalAll)
+		if err == nil {
+			if doc == nil {
+				doc = docInBucket.Body(ctx)
+			}
+		} else {
+			return false, err
+		}
+	}
+	shouldImport, err := importFilter.EvaluateFunction(ctx, doc, true)
+
+	return shouldImport, err
+}

db/import_test.go

@@ -446,47 +446,47 @@ func TestEvaluateFunction(t *testing.T) {
 	body := Body{"key": "value", "version": "1a"}
 	source := "illegal function(doc) {}"
 	importFilterFunc := NewImportFilterFunction(base.TestCtx(t), source, 0)
-	result, err := importFilterFunc.EvaluateFunction(base.TestCtx(t), body)
+	result, err := importFilterFunc.EvaluateFunction(base.TestCtx(t), body, false)
 	assert.Error(t, err, "Unexpected token function error")
 	assert.False(t, result, "Function evaluation result should be false")
 
 	// Simulate boolean return value from import filter function
 	body = Body{"key": "value", "version": "2a"}
 	source = `function(doc) { if (doc.version == "2a") { return true; } else { return false; }}`
 	importFilterFunc = NewImportFilterFunction(base.TestCtx(t), source, 0)
-	result, err = importFilterFunc.EvaluateFunction(base.TestCtx(t), body)
+	result, err = importFilterFunc.EvaluateFunction(base.TestCtx(t), body, false)
 	assert.NoError(t, err, "Import filter function shouldn't throw any error")
 	assert.True(t, result, "Import filter function should return boolean value true")
 
 	// Simulate non-boolean return value from import filter function; default switch case
 	body = Body{"key": "value", "version": "2b"}
 	source = `function(doc) { if (doc.version == "2b") { return 1.01; } else { return 0.01; }}`
 	importFilterFunc = NewImportFilterFunction(base.TestCtx(t), source, 0)
-	result, err = importFilterFunc.EvaluateFunction(base.TestCtx(t), body)
+	result, err = importFilterFunc.EvaluateFunction(base.TestCtx(t), body, false)
 	assert.Error(t, err, "Import filter function returned non-boolean value")
 	assert.False(t, result, "Import filter function evaluation result should be false")
 
 	// Simulate string return value true from import filter function
 	body = Body{"key": "value", "version": "1a"}
 	source = `function(doc) { if (doc.version == "1a") { return "true"; } else { return "false"; }}`
 	importFilterFunc = NewImportFilterFunction(base.TestCtx(t), source, 0)
-	result, err = importFilterFunc.EvaluateFunction(base.TestCtx(t), body)
+	result, err = importFilterFunc.EvaluateFunction(base.TestCtx(t), body, false)
 	assert.NoError(t, err, "Import filter function shouldn't throw any error")
 	assert.True(t, result, "Import filter function should return true")
 
 	// Simulate string return value false from import filter function
 	body = Body{"key": "value", "version": "2a"}
 	source = `function(doc) { if (doc.version == "1a") { return "true"; } else { return "false"; }}`
 	importFilterFunc = NewImportFilterFunction(base.TestCtx(t), source, 0)
-	result, err = importFilterFunc.EvaluateFunction(base.TestCtx(t), body)
+	result, err = importFilterFunc.EvaluateFunction(base.TestCtx(t), body, false)
 	assert.NoError(t, err, "Import filter function shouldn't throw any error")
 	assert.False(t, result, "Import filter function should return false")
 
 	// Simulate strconv.ParseBool: parsing "TruE": invalid syntax
 	body = Body{"key": "value", "version": "1a"}
 	source = `function(doc) { if (doc.version == "1a") { return "TruE"; } else { return "FaLsE"; }}`
 	importFilterFunc = NewImportFilterFunction(base.TestCtx(t), source, 0)
-	result, err = importFilterFunc.EvaluateFunction(base.TestCtx(t), body)
+	result, err = importFilterFunc.EvaluateFunction(base.TestCtx(t), body, false)
 	assert.Error(t, err, `strconv.ParseBool: parsing "TruE": invalid syntax`)
 	assert.False(t, result, "Import filter function should return true")
 }

docs/api/components/parameters.yaml

@@ -114,6 +114,14 @@ docid:
     type: string
   example: doc1
   description: The document ID to run the operation against.
+doc_id:
+  name: doc_id
+  in: query
+  required: false
+  schema:
+    type: string
+  example: doc1
+  description: The document ID to run the operation against.
 endkey:
   name: endkey
   in: query

docs/api/diagnostic.yaml

@@ -32,6 +32,10 @@ paths:
     $ref: ./paths/common/_ping.yaml
   '/{keyspace}/{docid}/_all_channels':
     $ref: './paths/diagnostic/keyspace-docid-_all_channels.yaml'
+  '/{keyspace}/sync':
+    $ref: './paths/diagnostic/keyspace-sync.yaml'
+  '/{keyspace}/import_filter':
+    $ref: './paths/diagnostic/keyspace-import_filter.yaml'
 externalDocs:
   description: Sync Gateway Quickstart | Couchbase Docs
   url: 'https://docs.couchbase.com/sync-gateway/current/index.html'

docs/api/paths/diagnostic/keyspace-import_filter.yaml

@@ -0,0 +1,42 @@
+# Copyright 2024-Present Couchbase, Inc.
+#
+# Use of this software is governed by the Business Source License included
+# in the file licenses/BSL-Couchbase.txt.  As of the Change Date specified
+# in that file, in accordance with the Business Source License, use of this
+# software will be governed by the Apache License, Version 2.0, included in
+# the file licenses/APL2.txt.
+parameters:
+  - $ref: ../../components/parameters.yaml#/keyspace
+get:
+  summary: Run a doc body through the Import filter and return results.
+  description: |-
+    Run a document body through the import filter and return whether its imported or not, and any error messages.
+    * Sync Gateway Application Read Only
+  requestBody:
+    content:
+      application/json:
+        schema:
+          $ref: ../../components/schemas.yaml#/Document
+  responses:
+    '200':
+      description: Document Processed by import filter successfully
+      content:
+        application/json:
+          schema:
+            type: object
+            properties:
+              shouldImport:
+                description: Whether this document would be imported after being processed by the import filter.
+                type: boolean
+              error:
+                description: Errors thrown by the Import filter.
+                type: string
+
+
+    '404':
+      $ref: ../../components/responses.yaml#/Not-found
+  parameters:
+    - $ref: ../../components/parameters.yaml#/doc_id
+  tags:
+    - Document
+  operationId: get_keyspace-import_filter-docid

docs/api/paths/diagnostic/keyspace-sync.yaml

@@ -0,0 +1,60 @@
+# Copyright 2024-Present Couchbase, Inc.
+#
+# Use of this software is governed by the Business Source License included
+# in the file licenses/BSL-Couchbase.txt.  As of the Change Date specified
+# in that file, in accordance with the Business Source License, use of this
+# software will be governed by the Apache License, Version 2.0, included in
+# the file licenses/APL2.txt.
+parameters:
+  - $ref: ../../components/parameters.yaml#/keyspace
+get:
+  summary: Run a doc body through the sync function and return sync data.
+  description: |-
+    Run a document body through the sync function and return document sync data.
+    * Sync Gateway Application Read Only
+  requestBody:
+    content:
+      application/json:
+        schema:
+          $ref: ../../components/schemas.yaml#/Document
+  responses:
+    '200':
+      description: Document Processed by sync function successfully
+      content:
+        application/json:
+          schema:
+            type: object
+            properties:
+              channels:
+                description: The channels the document was placed in by the sync function.
+                type: array
+              roles:
+                description: An access map of roles granted by the sync function.
+                type: object
+                properties:
+                  username:
+                    type: object
+                    additionalProperties:
+                      x-additionalPropertiesName: role
+                      type: string
+              access:
+                description: An access map of dynamic channels granted by the sync function.
+                type: object
+                properties:
+                  username:
+                    type: object
+                    additionalProperties:
+                      x-additionalPropertiesName: channel
+                      type: string
+
+              exception:
+                description: Errors thrown by the sync function.
+                type: string
+
+    '404':
+      $ref: ../../components/responses.yaml#/Not-found
+  parameters:
+    - $ref: ../../components/parameters.yaml#/doc_id
+  tags:
+    - Document
+  operationId: get_keyspace-sync