Skip to content

Commit

Permalink
CBG-3576: BlipTestClient support for HLV and rev tree modes (#6567)
Browse files Browse the repository at this point in the history 
* CBG-3210: Updating HLV on Put And PutExistingRev (#6366)

* CBG-3209: Add cv index and retrieval for revision cache (#6491)

* CBG-3209: changes for retreival of a doc from the rev cache via CV with backwards compatability in mind

* fix failing test, add commnets

* fix lint

* updated to address comments

* rebase chnages needed

* updated to tests that call Get on revision cache

* updates based of new direction with PR + addressing comments

* updated to fix panic

* updated to fix another panic

* address comments

* updates based off commnets

* remove commnented out line

* updates to skip test relying on import and update PutExistingRev doc update type to update HLV

* updates to remove code adding rev id to value inside addToRevMapPostLoad. Added code to assign this inside value.store

* remove redundent code

* CBG-3210: Updating HLV on Put And PutExistingRev (#6366)

* CBG-3209: Add cv index and retrieval for revision cache (#6491)

* CBG-3209: changes for retreival of a doc from the rev cache via CV with backwards compatability in mind

* fix failing test, add commnets

* fix lint

* updated to address comments

* rebase chnages needed

* updated to tests that call Get on revision cache

* updates based of new direction with PR + addressing comments

* updated to fix panic

* updated to fix another panic

* address comments

* updates based off commnets

* remove commnented out line

* updates to skip test relying on import and update PutExistingRev doc update type to update HLV

* updates to remove code adding rev id to value inside addToRevMapPostLoad. Added code to assign this inside value.store

* remove redundent code

* CBG-3576: changes to BlipTesterClient to run with version vector subprotocol and non version vector subprotocol

* updates to work with rebase changes

* changes to remove repeated code, assuming from the rebase?

* refactoring based off discussion with Ben

* lint error fix
  • Loading branch information
@gregns1 @mohammed-madi
gregns1 authored and mohammed-madi committed Apr 8, 2024
1 parent afcae7e commit c6e6860
Show file tree
Hide file tree
Showing 20 changed files with 1,183 additions and 260 deletions.
8 changes: 4 additions & 4 deletions db/access_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func TestDynamicChannelGrant(t *testing.T) {

// Create a document in channel chan1
doc1Body := Body{"channel": "chan1", "greeting": "hello"}
_, _, err = dbCollection.PutExistingRevWithBody(ctx, "doc1", doc1Body, []string{"1-a"}, false)
_, _, err = dbCollection.PutExistingRevWithBody(ctx, "doc1", doc1Body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV)
require.NoError(t, err)

// Verify user cannot access document
Expand All @@ -54,7 +54,7 @@ func TestDynamicChannelGrant(t *testing.T) {

// Write access granting document
grantingBody := Body{"type": "setaccess", "owner": "user1", "channel": "chan1"}
_, _, err = dbCollection.PutExistingRevWithBody(ctx, "grant1", grantingBody, []string{"1-a"}, false)
_, _, err = dbCollection.PutExistingRevWithBody(ctx, "grant1", grantingBody, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV)
require.NoError(t, err)

// Verify reloaded user can access document
Expand All @@ -66,12 +66,12 @@ func TestDynamicChannelGrant(t *testing.T) {

// Create a document in channel chan2
doc2Body := Body{"channel": "chan2", "greeting": "hello"}
_, _, err = dbCollection.PutExistingRevWithBody(ctx, "doc2", doc2Body, []string{"1-a"}, false)
_, _, err = dbCollection.PutExistingRevWithBody(ctx, "doc2", doc2Body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV)
require.NoError(t, err)

// Write access granting document for chan2 (tests invalidation when channels/inval_seq exists)
grantingBody = Body{"type": "setaccess", "owner": "user1", "channel": "chan2"}
_, _, err = dbCollection.PutExistingRevWithBody(ctx, "grant2", grantingBody, []string{"1-a"}, false)
_, _, err = dbCollection.PutExistingRevWithBody(ctx, "grant2", grantingBody, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV)
require.NoError(t, err)

// Verify user can now access both documents
Expand Down
28 changes: 14 additions & 14 deletions db/attachment_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ func TestBackupOldRevisionWithAttachments(t *testing.T) {
var rev2Body Body
rev2Data := `{"test": true, "updated": true, "_attachments": {"hello.txt": {"stub": true, "revpos": 1}}}`
require.NoError(t, base.JSONUnmarshal([]byte(rev2Data), &rev2Body))
_, _, err = collection.PutExistingRevWithBody(ctx, docID, rev2Body, []string{"2-abc", rev1ID}, true)
_, _, err = collection.PutExistingRevWithBody(ctx, docID, rev2Body, []string{"2-abc", rev1ID}, true, ExistingVersionWithUpdateToHLV)
require.NoError(t, err)
rev2ID := "2-abc"

Expand Down Expand Up @@ -196,7 +196,7 @@ func TestAttachments(t *testing.T) {
rev2Bstr := `{"_attachments": {"bye.txt": {"stub":true,"revpos":1,"digest":"sha1-gwwPApfQR9bzBKpqoEYwFmKp98A="}}, "_rev": "2-f000"}`
var body2B Body
assert.NoError(t, base.JSONUnmarshal([]byte(rev2Bstr), &body2B))
_, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body2B, []string{"2-f000", rev1id}, false)
_, _, err = collection.PutExistingRevWithBody(ctx, "doc1", body2B, []string{"2-f000", rev1id}, false, ExistingVersionWithUpdateToHLV)
assert.NoError(t, err, "Couldn't update document")
}

Expand Down Expand Up @@ -280,7 +280,7 @@ func TestAttachmentCASRetryAfterNewAttachment(t *testing.T) {
rev2Data := `{"prop1":"value2", "_attachments": {"hello.txt": {"data":"aGVsbG8gd29ybGQ="}}}`
require.NoError(t, base.JSONUnmarshal([]byte(rev2Data), &rev2Body))
collection := GetSingleDatabaseCollectionWithUser(t, db)
_, _, err := collection.PutExistingRevWithBody(ctx, "doc1", rev2Body, []string{"2-abc", rev1ID}, true)
_, _, err := collection.PutExistingRevWithBody(ctx, "doc1", rev2Body, []string{"2-abc", rev1ID}, true, ExistingVersionWithUpdateToHLV)
require.NoError(t, err)

log.Printf("Done creating rev 2 for key %s", key)
Expand Down Expand Up @@ -311,7 +311,7 @@ func TestAttachmentCASRetryAfterNewAttachment(t *testing.T) {
var rev3Body Body
rev3Data := `{"prop1":"value3", "_attachments": {"hello.txt": {"revpos":2,"stub":true,"digest":"sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0="}}}`
require.NoError(t, base.JSONUnmarshal([]byte(rev3Data), &rev3Body))
_, _, err = collection.PutExistingRevWithBody(ctx, "doc1", rev3Body, []string{"3-abc", "2-abc", rev1ID}, true)
_, _, err = collection.PutExistingRevWithBody(ctx, "doc1", rev3Body, []string{"3-abc", "2-abc", rev1ID}, true, ExistingVersionWithUpdateToHLV)
require.NoError(t, err)

log.Printf("rev 3 done")
Expand Down Expand Up @@ -343,7 +343,7 @@ func TestAttachmentCASRetryDuringNewAttachment(t *testing.T) {
rev2Data := `{"prop1":"value2"}`
require.NoError(t, base.JSONUnmarshal([]byte(rev2Data), &rev2Body))
collection := GetSingleDatabaseCollectionWithUser(t, db)
_, _, err := collection.PutExistingRevWithBody(ctx, "doc1", rev2Body, []string{"2-abc", rev1ID}, true)
_, _, err := collection.PutExistingRevWithBody(ctx, "doc1", rev2Body, []string{"2-abc", rev1ID}, true, ExistingVersionWithUpdateToHLV)
require.NoError(t, err)

log.Printf("Done creating rev 2 for key %s", key)
Expand Down Expand Up @@ -374,7 +374,7 @@ func TestAttachmentCASRetryDuringNewAttachment(t *testing.T) {
var rev3Body Body
rev3Data := `{"prop1":"value3", "_attachments": {"hello.txt": {"data":"aGVsbG8gd29ybGQ="}}}`
require.NoError(t, base.JSONUnmarshal([]byte(rev3Data), &rev3Body))
_, _, err = collection.PutExistingRevWithBody(ctx, "doc1", rev3Body, []string{"3-abc", "2-abc", rev1ID}, true)
_, _, err = collection.PutExistingRevWithBody(ctx, "doc1", rev3Body, []string{"3-abc", "2-abc", rev1ID}, true, ExistingVersionWithUpdateToHLV)
require.NoError(t, err)

log.Printf("rev 3 done")
Expand Down Expand Up @@ -563,57 +563,57 @@ func TestRetrieveAncestorAttachments(t *testing.T) {
// Create document (rev 1)
text := `{"key": "value", "version": "1a"}`
assert.NoError(t, base.JSONUnmarshal([]byte(text), &body))
doc, revID, err := collection.PutExistingRevWithBody(ctx, "doc", body, []string{"1-a"}, false)
doc, revID, err := collection.PutExistingRevWithBody(ctx, "doc", body, []string{"1-a"}, false, ExistingVersionWithUpdateToHLV)
assert.NoError(t, err, "Couldn't create document")
log.Printf("doc: %v", doc)

// Add an attachment to a document (rev 2)
text = `{"key": "value", "version": "2a", "_attachments": {"att1.txt": {"data": "YXR0MS50eHQ="}}}`
assert.NoError(t, base.JSONUnmarshal([]byte(text), &body))
body[BodyRev] = revID
doc, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"2-a", "1-a"}, false)
doc, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"2-a", "1-a"}, false, ExistingVersionWithUpdateToHLV)
assert.NoError(t, err, "Couldn't create document")
log.Printf("doc: %v", doc)

text = `{"key": "value", "version": "3a", "_attachments": {"att1.txt": {"stub":true,"revpos":2,"digest":"sha1-gwwPApfQR9bzBKpqoEYwFmKp98A="}}}`
assert.NoError(t, base.JSONUnmarshal([]byte(text), &body))
body[BodyRev] = revID
doc, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"3-a", "2-a"}, false)
doc, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"3-a", "2-a"}, false, ExistingVersionWithUpdateToHLV)
assert.NoError(t, err, "Couldn't create document")
log.Printf("doc: %v", doc)

text = `{"key": "value", "version": "4a", "_attachments": {"att1.txt": {"stub":true,"revpos":2,"digest":"sha1-gwwPApfQR9bzBKpqoEYwFmKp98A="}}}`
assert.NoError(t, base.JSONUnmarshal([]byte(text), &body))
body[BodyRev] = revID
doc, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"4-a", "3-a"}, false)
doc, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"4-a", "3-a"}, false, ExistingVersionWithUpdateToHLV)
assert.NoError(t, err, "Couldn't create document")
log.Printf("doc: %v", doc)

text = `{"key": "value", "version": "5a", "_attachments": {"att1.txt": {"stub":true,"revpos":2,"digest":"sha1-gwwPApfQR9bzBKpqoEYwFmKp98A="}}}`
assert.NoError(t, base.JSONUnmarshal([]byte(text), &body))
body[BodyRev] = revID
doc, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"5-a", "4-a"}, false)
doc, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"5-a", "4-a"}, false, ExistingVersionWithUpdateToHLV)
assert.NoError(t, err, "Couldn't create document")
log.Printf("doc: %v", doc)

text = `{"key": "value", "version": "6a", "_attachments": {"att1.txt": {"stub":true,"revpos":2,"digest":"sha1-gwwPApfQR9bzBKpqoEYwFmKp98A="}}}`
assert.NoError(t, base.JSONUnmarshal([]byte(text), &body))
body[BodyRev] = revID
doc, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"6-a", "5-a"}, false)
doc, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"6-a", "5-a"}, false, ExistingVersionWithUpdateToHLV)
assert.NoError(t, err, "Couldn't create document")
log.Printf("doc: %v", doc)

text = `{"key": "value", "version": "3b", "type": "pruned"}`
assert.NoError(t, base.JSONUnmarshal([]byte(text), &body))
body[BodyRev] = revID
doc, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"3-b", "2-a"}, false)
doc, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"3-b", "2-a"}, false, ExistingVersionWithUpdateToHLV)
assert.NoError(t, err, "Couldn't create document")
log.Printf("doc: %v", doc)

text = `{"key": "value", "version": "3b", "_attachments": {"att1.txt": {"stub":true,"revpos":2,"digest":"sha1-gwwPApfQR9bzBKpqoEYwFmKp98A="}}}`
assert.NoError(t, base.JSONUnmarshal([]byte(text), &body))
body[BodyRev] = revID
doc, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"3-b", "2-a"}, false)
doc, _, err = collection.PutExistingRevWithBody(ctx, "doc", body, []string{"3-b", "2-a"}, false, ExistingVersionWithUpdateToHLV)
assert.NoError(t, err, "Couldn't create document")
log.Printf("doc: %v", doc)
}
Expand Down
4 changes: 2 additions & 2 deletions db/blip_handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1210,9 +1210,9 @@ func (bh *blipHandler) processRev(rq *blip.Message, stats *processRevStats) (err
// bh.conflictResolver != nil represents an active SGR2 and BLIPClientTypeSGR2 represents a passive SGR2
forceAllowConflictingTombstone := newDoc.Deleted && (bh.conflictResolver != nil || bh.clientType == BLIPClientTypeSGR2)
if bh.conflictResolver != nil {
_, _, err = bh.collection.PutExistingRevWithConflictResolution(bh.loggingCtx, newDoc, history, true, bh.conflictResolver, forceAllowConflictingTombstone, rawBucketDoc)
_, _, err = bh.collection.PutExistingRevWithConflictResolution(bh.loggingCtx, newDoc, history, true, bh.conflictResolver, forceAllowConflictingTombstone, rawBucketDoc, ExistingVersionWithUpdateToHLV)
} else {
_, _, err = bh.collection.PutExistingRev(bh.loggingCtx, newDoc, history, revNoConflicts, forceAllowConflictingTombstone, rawBucketDoc)
_, _, err = bh.collection.PutExistingRev(bh.loggingCtx, newDoc, history, revNoConflicts, forceAllowConflictingTombstone, rawBucketDoc, ExistingVersionWithUpdateToHLV)
}
if err != nil {
return err
Expand Down
2 changes: 1 addition & 1 deletion db/change_cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -497,7 +497,7 @@ func (c *changeCache) DocChanged(event sgbucket.FeedEvent) {

// Now add the entry for the new doc revision:
if len(rawUserXattr) > 0 {
collection.revisionCache.Remove(docID, syncData.CurrentRev)
collection.revisionCache.RemoveWithRev(docID, syncData.CurrentRev)
}
change := &LogEntry{
Sequence: syncData.Sequence,
Expand Down
4 changes: 2 additions & 2 deletions db/changes_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -478,14 +478,14 @@ func BenchmarkChangesFeedDocUnmarshalling(b *testing.B) {

// Create child rev 1
docBody["child"] = "A"
_, _, err = collection.PutExistingRevWithBody(ctx, docid, docBody, []string{"2-A", revId}, false)
_, _, err = collection.PutExistingRevWithBody(ctx, docid, docBody, []string{"2-A", revId}, false, ExistingVersionWithUpdateToHLV)
if err != nil {
b.Fatalf("Error creating child1 rev: %v", err)
}

// Create child rev 2
docBody["child"] = "B"
_, _, err = collection.PutExistingRevWithBody(ctx, docid, docBody, []string{"2-B", revId}, false)
_, _, err = collection.PutExistingRevWithBody(ctx, docid, docBody, []string{"2-B", revId}, false, ExistingVersionWithUpdateToHLV)
if err != nil {
b.Fatalf("Error creating child2 rev: %v", err)
}
Expand Down
16 changes: 16 additions & 0 deletions db/database.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,15 @@ const (
DBCompactRunning
)

const (
Import DocUpdateType = iota
NewVersion
ExistingVersion
ExistingVersionWithUpdateToHLV
)

type DocUpdateType uint32

const (
DefaultRevsLimitNoConflicts = 50
DefaultRevsLimitConflicts = 100
Expand Down Expand Up @@ -88,6 +97,7 @@ type DatabaseContext struct {
MetadataStore base.DataStore // Storage for database metadata (anything that isn't an end-user's/customer's documents)
Bucket base.Bucket // Storage
BucketSpec base.BucketSpec // The BucketSpec
BucketUUID string // The bucket UUID for the bucket the database is created against
BucketLock sync.RWMutex // Control Access to the underlying bucket object
mutationListener changeListener // Caching feed listener
ImportListener *importListener // Import feed listener
Expand Down Expand Up @@ -398,6 +408,11 @@ func NewDatabaseContext(ctx context.Context, dbName string, bucket base.Bucket,
metadataStore = bucket.DefaultDataStore()
}

bucketUUID, err := bucket.UUID()
if err != nil {
return nil, err
}

// Register the cbgt pindex type for the configGroup
RegisterImportPindexImpl(ctx, options.GroupID)

Expand All @@ -406,6 +421,7 @@ func NewDatabaseContext(ctx context.Context, dbName string, bucket base.Bucket,
UUID: cbgt.NewUUID(),
MetadataStore: metadataStore,
Bucket: bucket,
BucketUUID: bucketUUID,
StartTime: time.Now(),
autoImport: autoImport,
Options: options,
Expand Down
Loading

0 comments on commit c6e6860

Please sign in to comment.