Merge pull request #22 from btnguyen2k/feature_subpartitions

Prepare to release v0.3.0

.github/workflows/gocosmos.yaml

@@ -195,7 +195,7 @@ jobs:
  - name: Test
  run: |
  export COSMOSDB_URL="AccountEndpoint=https://127.0.0.1:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw=="
- go test -cover -coverprofile="coverage_other.txt" -v -count 1 -p 1 -run "TestNew|TestStmt_NumInput|TestDriver_" .
+ go test -cover -coverprofile="coverage_other.txt" -v -count 1 -p 1 -run "TestNew|TestDriver_" .
  - name: Codecov
  uses: codecov/codecov-action@v3
  with:

README.md

@@ -20,7 +20,7 @@ Summary of supported SQL statements:
 |Delete an existing database |`DROP DATABASE [IF EXISTS] <db-name>`|
 |List all existing databases |`LIST DATABASES`|
 |Create a new collection |`CREATE COLLECTION [IF NOT EXISTS] [<db-name>.]<collection-name> <WITH [LARGE]PK=partitionKey>`|
-|Change collection's throughput |`ALTER COLLECTION [<db-name>.]<collection-name> WITH  RU/MAXRU=<ru>`|
+|Change collection's throughput |`ALTER COLLECTION [<db-name>.]<collection-name> WITH RU|MAXRU=<ru>`|
 |Delete an existing collection |`DROP COLLECTION [IF EXISTS] [<db-name>.]<collection-name>`|
 |List all existing collections in a database|`LIST COLLECTIONS [FROM <db-name>]`|
 |Insert a new document into collection |`INSERT INTO [<db-name>.]<collection-name> ...`|
@@ -77,7 +77,7 @@ AccountEndpoint=<cosmosdb-endpoint>
 - `AccountEndpoint`: (required) endpoint to access Cosmos DB. For example, the endpoint for Azure Cosmos DB Emulator running on local is `https://localhost:8081/`.
 - `AccountKey`: (required) account key to authenticate.
 - `TimeoutMs`: (optional) operation timeout in milliseconds. Default value is `10 seconds` if not specified.
-- `Version`: (optional) version of Cosmos DB to use. Default value is `2018-12-31` if not specified. See: https://learn.microsoft.com/rest/api/cosmos-db/#supported-rest-api-versions.
+- `Version`: (optional) version of Cosmos DB to use. Default value is `2020-07-15` if not specified. See: https://learn.microsoft.com/rest/api/cosmos-db/#supported-rest-api-versions.
 - `DefaultDb`: (optional, available since [v0.1.1](RELEASE-NOTES.md)) specify the default database used in Cosmos DB operations. Alias `Db` can also be used instead of `DefaultDb`.
 - `AutoId`: (optional, available since [v0.1.2](RELEASE-NOTES.md)) see [auto id](#auto-id) session.
 - `InsecureSkipVerify`: (optional, available since [v0.1.4](RELEASE-NOTES.md)) if `true`, disable CA verification for https endpoint (useful to run against test/dev env with local/docker Cosmos DB emulator).
@@ -156,7 +156,7 @@ AccountEndpoint=<cosmosdb-endpoint>
 - `AccountEndpoint`: (required) endpoint to access Cosmos DB. For example, the endpoint for Azure Cosmos DB Emulator running on local is `https://localhost:8081/`.
 - `AccountKey`: (required) account key to authenticate.
 - `TimeoutMs`: (optional) operation timeout in milliseconds. Default value is `10 seconds` if not specified.
-- `Version`: (optional) version of Cosmos DB to use. Default value is `2018-12-31` if not specified. See: https://learn.microsoft.com/rest/api/cosmos-db/#supported-rest-api-versions.
+- `Version`: (optional) version of Cosmos DB to use. Default value is `2020-07-15` if not specified. See: https://learn.microsoft.com/rest/api/cosmos-db/#supported-rest-api-versions.
 - `AutoId`: (optional, available since [v0.1.2](RELEASE-NOTES.md)) see [auto id](#auto-id) session.
 - `InsecureSkipVerify`: (optional, available since [v0.1.4](RELEASE-NOTES.md)) if `true`, disable CA verification for https endpoint (useful to run against test/dev env with local/docker Cosmos DB emulator).
 

RELEASE-NOTES.md

@@ -1,5 +1,11 @@
 # gocosmos - Release notes
 
+## 2023-06-16 - v0.3.0
+
+- Change default API version to `2020-07-15`.
+- Add [Hierarchical Partition Keys](https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys) (sub-partitions) support.
+- Use PartitionKey version 2 (replacing version 1), hence large PK is always enabled.
+
 ## 2023-06-09 - v0.2.1
 
 - Bug fixes, Refactoring & Enhancements.

SQL.md

@@ -168,7 +168,9 @@ fmt.Println(dbresult.RowsAffected())
 
 - Upon successful execution, `RowsAffected()` returns `(1, nil)`.
 - This statement returns error `ErrConflict` if the specified collection already existed. If `IF NOT EXISTS` is specified, `RowsAffected()` returns `(0, nil)`.
-- Partition key must be specified using `WITH pk=<partition-key>`. If partition key is larger than 100 bytes, use `WITH pk=<partition-key>` instead.
+- Partition key must be specified using `WITH pk=<partition-key>`.
+ - Since [v0.3.0](RELEASE-NOTES.md), large pk is always enabled, `WITH largepk` is for backward compatibility only.
+ - Since [v0.3.0](RELEASE-NOTES.md), Hierarchical Partition Key is supported, using `WITH pk=/path1,/path2...` (up to 3 path levels).
 - Provisioned capacity can be optionally specified via `WITH RU=<ru>` or `WITH MAXRU=<ru>`.
  - Only one of `RU` and `MAXRU` options should be specified, _not both_; error is returned if both optiosn are specified.
 - Unique keys are optionally specified via `WITH uk=/uk1_path:/uk2_path1,/uk2_path2:/uk3_path`. Each unique key is a comma-separated list of paths (e.g. `/uk_path1,/uk_path2`); unique keys are separated by colons (e.g. `/uk1:/uk2:/uk3`).
@@ -295,11 +297,16 @@ Description: insert a new document into an existing collection.
 Syntax:
 
 ```sql
-INSERT INTO [<db-name>.]<collection-name> (<field1>, <field2>,...<fieldN>) VALUES (<value1>, <value2>,...<valueN>)
+INSERT INTO [<db-name>.]<collection-name>
+(<field1>, <field2>,...<fieldN>)
+VALUES (<value1>, <value2>,...<valueN>)
+[WITH singlePK|SINGLE_PK[=true]]
 ```
 
 > `<db-name>` can be ommitted if `DefaultDb` is supplied in the Data Source Name (DSN).
 
+Since [v0.3.0](RELEASE-NOTES.md), `gocosmos` supports [Hierarchical Partition Keys](https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys) (or sub-partitions). If the collection is known not to have sub-partitions, supplying `WITH singlePK` (or `WITH SINGLE_PK`) can save one roundtrip to Cosmos DB server.
+
 Example:
 ```go
 sql := `INSERT INTO mydb.mytable (a, b, c, d, e) VALUES (1, "\"a string\"", :1, @2, $3)`
@@ -312,8 +319,7 @@ fmt.Println(dbresult.RowsAffected())
 
 > Use `sql.DB.Exec` to execute the statement, `Query` will return error.
 
-> Value of partition key _must_ be supplied at the last argument of `db.Exec()` call.
-
+> Values of partition keys _must_ be supplied at the end of the argument list when invoking `db.Exec()`.
 
 <a id="value"></a>A value is either:
 - a placeholder - which is a number prefixed by `$` or `@` or `:`, for example `$1`, `@2` or `:3`. Placeholders are 1-based index, that means starting from 1.
@@ -337,7 +343,10 @@ Description: insert a new document or replace an existing one.
 Syntax & Usage: similar to [INSERT](#insert).
 
 ```sql
-UPSERT INTO [<db-name>.]<collection-name> (<field1>, <field2>,...<fieldN>) VALUES (<value1>, <value2>,...<valueN>)
+UPSERT INTO [<db-name>.]<collection-name>
+(<field1>, <field2>,...<fieldN>)
+VALUES (<value1>, <value2>,...<valueN>)
+[WITH singlePK|SINGLE_PK[=true]]
 ```
 
 [Back to top](#top)
@@ -349,11 +358,15 @@ Description: delete an existing document.
 Syntax:
 
 ```sql
-DELETE FROM [<db-name>.]<collection-name> WHERE id=<id-value>
+DELETE FROM [<db-name>.]<collection-name>
+WHERE id=<id-value>
+[WITH singlePK|SINGLE_PK[=true]]
 ```
 
 > `<db-name>` can be ommitted if `DefaultDb` is supplied in the Data Source Name (DSN).
 
+Since [v0.3.0](RELEASE-NOTES.md), `gocosmos` supports [Hierarchical Partition Keys](https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys) (or sub-partitions). If the collection is known not to have sub-partitions, supplying `WITH singlePK` (or `WITH SINGLE_PK`) can save one roundtrip to Cosmos DB server.
+
 Example:
 ```go
 sql := `DELETE FROM mydb.mytable WHERE id=@1`
@@ -366,7 +379,7 @@ fmt.Println(dbresult.RowsAffected())
 
 > Use `sql.DB.Exec` to execute the statement, `Query` will return error.
 
-> Value of partition key _must_ be supplied at the last argument of `db.Exec()` call.
+> Values of partition keys _must_ be supplied at the end of the argument list when invoking `db.Exec()`.
 
 - `DELETE` removes only one document specified by id.
 - Upon successful execution, `RowsAffected()` returns `(1, nil)`. If no document matched, `RowsAffected()` returns `(0, nil)`.
@@ -381,11 +394,16 @@ Description: update an existing document.
 Syntax:
 
 ```sql
-UPDATE [<db-name>.]<collection-name> SET <fiel1>=<value1>[,<field2>=<value2>,...<fieldN>=<valueN>] WHERE id=<id-value>
+UPDATE [<db-name>.]<collection-name>
+SET <fiel1>=<value1>[,<field2>=<value2>,...<fieldN>=<valueN>]
+WHERE id=<id-value>
+[WITH singlePK|SINGLE_PK[=true]]
 ```
 
 > `<db-name>` can be ommitted if `DefaultDb` is supplied in the Data Source Name (DSN).
 
+Since [v0.3.0](RELEASE-NOTES.md), `gocosmos` supports [Hierarchical Partition Keys](https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys) (or sub-partitions). If the collection is known not to have sub-partitions, supplying `WITH singlePK` (or `WITH SINGLE_PK`) can save one roundtrip to Cosmos DB server.
+
 Example:
 ```go
 sql := `UPDATE mydb.mytable SET a=1, b="\"a string\"", c=true, d="[1,true,null,\"string\"]", e=:2 WHERE id=@1`
@@ -398,7 +416,7 @@ fmt.Println(dbresult.RowsAffected())
 
 > Use `sql.DB.Exec` to execute the statement, `Query` will return error.
 
-> Value of partition key _must_ be supplied at the last argument of `db.Exec()` call.
+> Values of partition keys _must_ be supplied at the end of the argument list when invoking `db.Exec()`.
 
 - `UPDATE` modifies only one document specified by id.
 - Upon successful execution, `RowsAffected()` returns `(1, nil)`. If no document matched, `RowsAffected()` returns `(0, nil)`.
@@ -417,7 +435,7 @@ Syntax:
 SELECT [CROSS PARTITION] ... FROM <collection-name> ...
 [WITH database=<db-name>]
 [[,] WITH collection=<collection-name>]
-[[,] WITH cross_partition=true]
+[[,] WITH cross_partition|CrossPartition[=true]]
 ```
 
 > `<db-name>` can be ommitted if `DefaultDb` is supplied in the Data Source Name (DSN).

data_test.go

@@ -14,11 +14,72 @@ import (
 
 /*======================================================================*/
 
+const numApps = 4
 const numLogicalPartitions = 16
 const numCategories = 19
 
 var dataList []DocInfo
 
+func _initDataSubPartitions(t *testing.T, testName string, client *RestClient, db, container string, numItem int) {
+ totalRu := 0.0
+ randList := make([]int, numItem)
+ for i := 0; i < numItem; i++ {
+ randList[i] = i*2 + 1
+ }
+ rand.Shuffle(numItem, func(i, j int) {
+ randList[i], randList[j] = randList[j], randList[i]
+ })
+ dataList = make([]DocInfo, numItem)
+ for i := 0; i < numItem; i++ {
+ category := randList[i] % numCategories
+ app := "app" + strconv.Itoa(i%numApps)
+ username := "user" + strconv.Itoa(i%numLogicalPartitions)
+ docInfo := DocInfo{
+ "id": fmt.Sprintf("%05d", i),
+ "app": app,
+ "username": username,
+ "email": "user" + strconv.Itoa(i) + "@domain.com",
+ "grade": float64(randList[i]),
+ "category": float64(category),
+ "active": i%10 == 0,
+ "big": fmt.Sprintf("%05d", i) + "/" + strings.Repeat("this is a very long string/", 256),
+ }
+ dataList[i] = docInfo
+ if result := client.CreateDocument(DocumentSpec{DbName: db, CollName: container, PartitionKeyValues: []interface{}{app, username}, DocumentData: docInfo}); result.Error() != nil {
+ t.Fatalf("%s failed: %s", testName, result.Error())
+ } else {
+ totalRu += result.RequestCharge
+ }
+ }
+ // fmt.Printf("\t%s - total RU charged: %0.3f\n", testName+"/Insert", totalRu)
+}
+
+func _initDataSubPartitionsSmallRU(t *testing.T, testName string, client *RestClient, db, container string, numItem int) {
+ client.DeleteDatabase(db)
+ client.CreateDatabase(DatabaseSpec{Id: db, Ru: 400})
+ client.CreateCollection(CollectionSpec{
+ DbName: db,
+ CollName: container,
+ PartitionKeyInfo: map[string]interface{}{"paths": []string{"/app", "/username"}, "kind": "MultiHash", "version": 2},
+ UniqueKeyPolicy: map[string]interface{}{"uniqueKeys": []map[string]interface{}{{"paths": []string{"/email"}}}},
+ Ru: 400,
+ })
+ _initDataSubPartitions(t, testName, client, db, container, numItem)
+}
+
+func _initDataSubPartitionsLargeRU(t *testing.T, testName string, client *RestClient, db, container string, numItem int) {
+ client.DeleteDatabase(db)
+ client.CreateDatabase(DatabaseSpec{Id: db, Ru: 20000})
+ client.CreateCollection(CollectionSpec{
+ DbName: db,
+ CollName: container,
+ PartitionKeyInfo: map[string]interface{}{"paths": []string{"/app", "/username"}, "kind": "MultiHash", "version": 2},
+ UniqueKeyPolicy: map[string]interface{}{"uniqueKeys": []map[string]interface{}{{"paths": []string{"/email"}}}},
+ Ru: 20000,
+ })
+ _initDataSubPartitions(t, testName, client, db, container, numItem)
+}
+
 func _initData(t *testing.T, testName string, client *RestClient, db, container string, numItem int) {
  totalRu := 0.0
  randList := make([]int, numItem)

driver.go

@@ -98,7 +98,7 @@ type Driver struct {
 //
 // AccountEndpoint=<cosmosdb-restapi-endpoint>;AccountKey=<account-key>[;TimeoutMs=<timeout-in-ms>][;Version=<cosmosdb-api-version>][;DefaultDb=<db-name>][;AutoId=<true/false>][;InsecureSkipVerify=<true/false>]
 //
-// If not supplied, default value for TimeoutMs is 10 seconds, Version is defaultApiVersion (which is "2018-12-31"), AutoId is true, and InsecureSkipVerify is false
+// If not supplied, default value for TimeoutMs is 10 seconds, Version is DefaultApiVersion (which is "2020-07-15"), AutoId is true, and InsecureSkipVerify is false
 //
 // - DefaultDb is added since v0.1.1
 // - AutoId is added since v0.1.2

gocosmos.go

@@ -7,7 +7,7 @@ import (
 
 const (
  // Version of package gocosmos.
- Version = "0.2.1"
+ Version = "0.3.0"
 )
 
 func goTypeToCosmosDbType(typ reflect.Type) string {

restclient.go

@@ -31,7 +31,13 @@ const (
  settingVersion = "VERSION"
  settingAutoId = "AUTOID"
  settingInsecureSkipVerify = "INSECURESKIPVERIFY"
- defaultApiVersion = "2018-12-31"
+
+ // DefaultApiVersion holds the default REST API version if not specified in the connection string.
+ //
+ // See: https://learn.microsoft.com/en-us/rest/api/cosmos-db/#supported-rest-api-versions
+ //
+ // @Available since v0.3.0
+ DefaultApiVersion = "2020-07-15"
 )
 
 // NewRestClient constructs a new RestClient instance from the supplied connection string.
@@ -41,7 +47,7 @@ const (
 //
 // AccountEndpoint=<cosmosdb-restapi-endpoint>;AccountKey=<account-key>[;TimeoutMs=<timeout-in-ms>][;Version=<cosmosdb-api-version>][;AutoId=<true/false>][;InsecureSkipVerify=<true/false>]
 //
-// If not supplied, default value for TimeoutMs is 10 seconds, Version is defaultApiVersion (which is "2018-12-31"), AutoId is true, and InsecureSkipVerify is false
+// If not supplied, default value for TimeoutMs is 10 seconds, Version is DefaultApiVersion (which is "2020-07-15"), AutoId is true, and InsecureSkipVerify is false
 //
 // - AutoId is added since v0.1.2
 // - InsecureSkipVerify is added since v0.1.4
@@ -75,7 +81,7 @@ func NewRestClient(httpClient *http.Client, connStr string) (*RestClient, error)
  }
  apiVersion := params[settingVersion]
  if apiVersion == "" {
- apiVersion = defaultApiVersion
+ apiVersion = DefaultApiVersion
  }
  autoId, err := strconv.ParseBool(params[settingAutoId])
  if err != nil {
@@ -1194,6 +1200,35 @@ type RespListDb struct {
  Databases []DbInfo `json:"Databases"`
 }
 
+// PkInfo holds partitioning configuration settings for a collection.
+//
+// @Available since v0.3.0
+type PkInfo map[string]interface{}
+
+func (pk PkInfo) Kind() string {
+ kind, err := reddo.ToString(pk["kind"])
+ if err == nil {
+ return kind
+ }
+ return ""
+}
+
+func (pk PkInfo) Version() int {
+ version, err := reddo.ToInt(pk["version"])
+ if err == nil {
+ return int(version)
+ }
+ return 0
+}
+
+func (pk PkInfo) Paths() []string {
+ paths, err := reddo.ToSlice(pk["paths"], reddo.TypeString)
+ if err == nil {
+ return paths.([]string)
+ }
+ return nil
+}
+
 // CollInfo captures info of a Cosmos DB collection.
 type CollInfo struct {
  Id string `json:"id"` // user-generated unique name for the collection
@@ -1207,7 +1242,7 @@ type CollInfo struct {
  Udfs string `json:"_udfs"` // (system-generated property) _udfs attribute of the collection
  Conflicts string `json:"_conflicts"` // (system-generated property) _conflicts attribute of the collection
  IndexingPolicy map[string]interface{} `json:"indexingPolicy"` // indexing policy settings for collection
- PartitionKey map[string]interface{} `json:"partitionKey"` // partitioning configuration settings for collection
+ PartitionKey PkInfo  `json:"partitionKey"` // partitioning configuration settings for collection
  ConflictResolutionPolicy map[string]interface{} `json:"conflictResolutionPolicy"` // conflict resolution policy settings for collection
  GeospatialConfig map[string]interface{} `json:"geospatialConfig"` // Geo-spatial configuration settings for collection
 }

restclient_collection_test.go

@@ -65,6 +65,54 @@ func TestRestClient_CreateCollection(t *testing.T) {
  }
 }
 
+func TestRestClient_CreateCollection_SubPartitions(t *testing.T) {
+ name := "TestRestClient_CreateCollection_SubPartitions"
+ client := _newRestClient(t, name)
+
+ dbname := testDb
+ collname := testTable
+ collspecList := []CollectionSpec{
+ // Hierarchical Partition Keys
+ {DbName: dbname, CollName: collname, PartitionKeyInfo: map[string]interface{}{"paths": []string{"/TenantId", "/UserId"}, "kind": "MultiHash", "version": 2}},
+ {DbName: dbname, CollName: collname, MaxRu: 4000, PartitionKeyInfo: map[string]interface{}{"paths": []string{"/TenantId", "/UserId", "/SessionId"}, "kind": "MultiHash", "version": 2}},
+ }
+ for _, collspec := range collspecList {
+ client.DeleteDatabase(dbname)
+ client.CreateDatabase(DatabaseSpec{Id: dbname})
+ var collInfo CollInfo
+ if result := client.CreateCollection(collspec); result.Error() != nil {
+ t.Fatalf("%s failed: %s", name, result.Error())
+ } else if result.Id != collname {
+ t.Fatalf("%s failed: <coll-id> expected %#v but received %#v", name+"/CreateDatabase", collname, result.Id)
+ } else if result.Rid == "" || result.Self == "" || result.Etag == "" || result.Docs == "" ||
+ result.Sprocs == "" || result.Triggers == "" || result.Udfs == "" || result.Conflicts == "" ||
+ result.Ts <= 0 || len(result.IndexingPolicy) == 0 || len(result.PartitionKey) == 0 {
+ t.Fatalf("%s failed: invalid collinfo returned %#v", name, result.CollInfo)
+ } else {
+ collInfo = result.CollInfo
+ }
+
+ if collspec.Ru > 0 || collspec.MaxRu > 0 {
+ if result := client.GetOfferForResource(collInfo.Rid); result.Error() != nil {
+ t.Fatalf("%s failed: %s", name, result.Error())
+ } else {
+ if ru, maxru := result.OfferThroughput(), result.MaxThroughputEverProvisioned(); collspec.Ru > 0 && (collspec.Ru != ru || collspec.Ru != maxru) {
+ t.Fatalf("%s failed: <offer-throughput> expected %#v but expected {ru:%#v, maxru:%#v}", name, collspec.Ru, ru, maxru)
+ }
+ if ru, maxru := result.OfferThroughput(), result.MaxThroughputEverProvisioned(); collspec.MaxRu > 0 && (collspec.MaxRu != ru*10 || collspec.MaxRu != maxru) {
+ t.Fatalf("%s failed: <max-throughput> expected %#v but expected {ru:%#v, maxru:%#v}", name, collspec.MaxRu, ru, maxru)
+ }
+ }
+ }
+
+ if result := client.CreateCollection(collspec); result.CallErr != nil {
+ t.Fatalf("%s failed: %s", name, result.CallErr)
+ } else if result.StatusCode != 409 {
+ t.Fatalf("%s failed: <status-code> expected %#v but received %#v", name, 409, result.StatusCode)
+ }
+ }
+}
+
 func TestRestClient_ChangeOfferCollection(t *testing.T) {
  name := "TestRestClient_ChangeOfferCollection"
  client := _newRestClient(t, name)