From ff3b04048c114cd4f3f76f1d771994b794d7077a Mon Sep 17 00:00:00 2001 From: xiaoweim Date: Thu, 7 Nov 2024 05:52:20 +0000 Subject: [PATCH] migration bigquerydataset to new API --- go.mod | 10 +- go.sum | 31 +- mockgcp/mockbigquery/datasets.go | 37 +++ .../bigquerydataset_mappings.go | 279 ++++++++++++------ .../bigquerydataset/dataset_controller.go | 148 ++++++---- .../direct/bigquerydataset/utils.go | 47 +++ pkg/controller/direct/maputils.go | 10 + .../basicbigquerydataset-direct/_http.log | 23 +- .../_http.log | 23 +- ..._export_fullybigquerydataset-direct.golden | 1 + .../fullybigquerydataset-direct/_http.log | 27 +- tests/e2e/normalize.go | 7 + .../bigquery/dataset/set_unset/_http05.log | 67 +---- .../bigquery/dataset/set_unset/_http06.log | 67 +---- .../bigquery/dataset/set_unset/_http07.log | 67 +---- .../bigquery/dataset/set_unset/_object05.yaml | 10 +- .../bigquery/dataset/set_unset/_object06.yaml | 10 +- .../bigquery/dataset/set_unset/_object07.yaml | 10 +- .../bigquery/dataset/set_unset/script.yaml | 6 + tests/e2e/unified_test.go | 7 + 20 files changed, 492 insertions(+), 395 deletions(-) diff --git a/go.mod b/go.mod index 676d8cf31d..938842e6de 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,6 @@ require ( github.com/hashicorp/hcl/v2 v2.19.1 github.com/hashicorp/terraform-plugin-sdk/v2 v2.24.0 github.com/hashicorp/terraform-provider-google-beta v3.73.0+incompatible - github.com/huandu/go-clone v1.7.2 github.com/nasa9084/go-openapi v0.0.0-20200604141640-2875b7376353 github.com/olekukonko/tablewriter v0.0.5 github.com/onsi/gomega v1.27.10 @@ -103,6 +102,7 @@ require ( github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect github.com/agext/levenshtein v1.2.3 // indirect + github.com/apache/arrow/go/v15 v15.0.2 // indirect github.com/apparentlymart/go-cidr v1.1.0 // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect @@ -136,11 +136,13 @@ require ( github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/gobuffalo/flect v0.2.3 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v1.2.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.3 // indirect + github.com/google/flatbuffers v23.5.26+incompatible // indirect github.com/google/gnostic v0.6.9 // indirect github.com/google/go-cpy v0.0.0-20211218193943-a9c933c06932 // indirect github.com/google/gofuzz v1.2.0 // indirect @@ -177,6 +179,8 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -200,6 +204,7 @@ require ( github.com/oklog/run v1.0.0 // indirect github.com/onsi/ginkgo v1.16.5 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/prometheus/client_model v0.6.0 // indirect @@ -214,6 +219,7 @@ require ( github.com/vmihailenco/tagparser v0.1.2 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xlab/treeprint v1.1.0 // indirect + github.com/zeebo/xxh3 v1.0.2 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect go.opentelemetry.io/otel v1.29.0 // indirect @@ -224,12 +230,14 @@ require ( go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect go.uber.org/multierr v1.10.0 // indirect golang.org/x/crypto v0.29.0 // indirect + golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect golang.org/x/mod v0.20.0 // indirect golang.org/x/net v0.31.0 // indirect golang.org/x/sys v0.27.0 // indirect golang.org/x/term v0.26.0 // indirect golang.org/x/text v0.20.0 // indirect golang.org/x/tools v0.24.0 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect google.golang.org/appengine v1.6.8 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index fbe7dcd516..4efe14472a 100644 --- a/go.sum +++ b/go.sum @@ -48,6 +48,8 @@ cloud.google.com/go/compute v1.29.0 h1:Lph6d8oPi38NHkOr6S55Nus/Pbbcp37m/J0ohgKAe cloud.google.com/go/compute v1.29.0/go.mod h1:HFlsDurE5DpQZClAGf/cYh+gxssMhBxBovZDYkEn/Og= cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo= cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k= +cloud.google.com/go/datacatalog v1.23.0 h1:9F2zIbWNNmtrSkPIyGRQNsIugG5VgVVFip6+tXSdWLg= +cloud.google.com/go/datacatalog v1.23.0/go.mod h1:9Wamq8TDfL2680Sav7q3zEhBJSPBrDxJU8WtPJ25dBM= cloud.google.com/go/dataflow v0.10.2 h1:o9P5/zR2mOYJmCnfp9/7RprKFZCwmSu3TvemQSmCaFM= cloud.google.com/go/dataflow v0.10.2/go.mod h1:+HIb4HJxDCZYuCqDGnBHZEglh5I0edi/mLgVbxDf0Ag= cloud.google.com/go/dataform v0.10.2 h1:t16DoejuOHoxJR88qrpdmFFlCXA9+x5PKrqI9qiDYz0= @@ -152,6 +154,8 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYU github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/apache/arrow/go/v15 v15.0.2 h1:60IliRbiyTWCWjERBCkO1W4Qun9svcYoZrSLcyOsMLE= +github.com/apache/arrow/go/v15 v15.0.2/go.mod h1:DGXsR3ajT524njufqf95822i+KTh+yea1jass9YXgjA= github.com/apparentlymart/go-cidr v1.1.0 h1:2mAhrMoF+nhXqxTzSZMUzDHkLjmIHC+Zzn4tdgBZjnU= github.com/apparentlymart/go-cidr v1.1.0/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc= github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM= @@ -405,6 +409,8 @@ github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= github.com/gobuffalo/flect v0.2.3 h1:f/ZukRnSNA/DUpSNDadko7Qc0PhGvsew35p/2tu+CRY= github.com/gobuffalo/flect v0.2.3/go.mod h1:vmkQwuZYhN5Pc4ljYQZzP+1sq+NEkK+lh20jmEmX3jc= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -459,6 +465,8 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= +github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -484,9 +492,12 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -614,10 +625,6 @@ github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734/go.mod github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d h1:kJCB4vdITiW1eC1vq2e6IsrXKrZit1bv/TDYFGMp4BQ= github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huandu/go-assert v1.1.5 h1:fjemmA7sSfYHJD7CUqs9qTwwfdNAx7/j2/ZlHXzNB3c= -github.com/huandu/go-assert v1.1.5/go.mod h1:yOLvuqZwmcHIC5rIzrBhT7D3Q9c3GFnd0JrPVhn/06U= -github.com/huandu/go-clone v1.7.2 h1:3+Aq0Ed8XK+zKkLjE2dfHg0XrpIfcohBE1K+c8Usxoo= -github.com/huandu/go-clone v1.7.2/go.mod h1:ReGivhG6op3GYr+UY3lS6mxjKp7MIGTknuU5TbTVaXE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -658,6 +665,10 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -792,6 +803,8 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= +github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -952,6 +965,10 @@ github.com/zclconf/go-cty v1.10.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uU github.com/zclconf/go-cty v1.13.0 h1:It5dfKTTZHe9aeppbNOda3mN7Ag7sg6QkBNm6TkyFa0= github.com/zclconf/go-cty v1.13.0/go.mod h1:YKQzy/7pZ7iq2jNFzy5go57xdxdWoLLpaEp4u238AE0= github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= @@ -1042,6 +1059,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1352,10 +1371,14 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= gomodules.xyz/jsonpatch/v2 v2.3.0 h1:8NFhfS6gzxNqjLIYnZxg319wZ5Qjnx4m/CcX+Klzazc= gomodules.xyz/jsonpatch/v2 v2.3.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= +gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= +gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= diff --git a/mockgcp/mockbigquery/datasets.go b/mockgcp/mockbigquery/datasets.go index 6dc1d09efe..e158f22872 100644 --- a/mockgcp/mockbigquery/datasets.go +++ b/mockgcp/mockbigquery/datasets.go @@ -209,6 +209,43 @@ func (s *datasetsServer) UpdateDataset(ctx context.Context, req *pb.UpdateDatase return updated, err } +func (s *datasetsServer) PatchDataset(ctx context.Context, req *pb.PatchDatasetRequest) (*pb.Dataset, error) { + name, err := s.buildDatasetName(req.GetProjectId(), req.GetDatasetId()) + if err != nil { + return nil, err + } + + fqn := name.String() + + existing := &pb.Dataset{} + if err := s.storage.Get(ctx, fqn, existing); err != nil { + return nil, err + } + + now := time.Now() + + updated := req.GetDataset() + updated.DatasetReference = existing.DatasetReference + + updated.CreationTime = existing.CreationTime + updated.LastModifiedTime = PtrTo(now.UnixMilli()) + updated.Id = PtrTo(existing.GetDatasetReference().GetProjectId() + ":" + existing.GetDatasetReference().GetDatasetId()) + updated.Kind = PtrTo("bigquery#dataset") + updated.Location = existing.Location + updated.Type = existing.Type + updated.SelfLink = PtrTo("https://bigquery.googleapis.com/bigquery/v2/" + name.String()) + + sortAccess(updated) + + updated.Etag = PtrTo(computeEtag(updated)) + + if err := s.storage.Update(ctx, fqn, updated); err != nil { + return nil, err + } + + return updated, err +} + func (s *datasetsServer) DeleteDataset(ctx context.Context, req *pb.DeleteDatasetRequest) (*empty.Empty, error) { name, err := s.buildDatasetName(req.GetProjectId(), req.GetDatasetId()) if err != nil { diff --git a/pkg/controller/direct/bigquerydataset/bigquerydataset_mappings.go b/pkg/controller/direct/bigquerydataset/bigquerydataset_mappings.go index 14f5462b9c..f9dd1ba9e9 100644 --- a/pkg/controller/direct/bigquerydataset/bigquerydataset_mappings.go +++ b/pkg/controller/direct/bigquerydataset/bigquerydataset_mappings.go @@ -21,210 +21,309 @@ package bigquerydataset import ( + "fmt" + "slices" "strconv" + "strings" + "time" + pb "cloud.google.com/go/bigquery" krm "github.com/GoogleCloudPlatform/k8s-config-connector/apis/bigquery/v1beta1" "github.com/GoogleCloudPlatform/k8s-config-connector/apis/refs/v1beta1" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/direct" - api "google.golang.org/api/bigquery/v2" ) -func BigQueryDatasetSpec_ToAPI(mapCtx *direct.MapContext, in *krm.BigQueryDatasetSpec, name string) *api.Dataset { +func BigQueryDatasetSpec_ToProto(mapCtx *direct.MapContext, in *krm.BigQueryDatasetSpec) *pb.DatasetMetadata { if in == nil { return nil } - out := &api.Dataset{} - acccessList := []*api.DatasetAccess{} + out := &pb.DatasetMetadata{} + acccessList := []*pb.AccessEntry{} for _, access := range in.Access { - curAccess := Access_ToAPI(mapCtx, direct.LazyPtr(access)) + curAccess := AccessEntry_ToProto(mapCtx, direct.LazyPtr(access)) acccessList = append(acccessList, curAccess) } out.Access = acccessList out.DefaultCollation = direct.ValueOf(in.DefaultCollation) - out.DefaultPartitionExpirationMs = direct.ValueOf(in.DefaultPartitionExpirationMs) - out.DefaultTableExpirationMs = direct.ValueOf(in.DefaultTableExpirationMs) - out.DefaultEncryptionConfiguration = EncryptionConfiguration_ToAPI(mapCtx, in.DefaultEncryptionConfiguration) + out.DefaultPartitionExpiration = time.Duration(direct.ValueOf(in.DefaultPartitionExpirationMs)) * time.Millisecond + out.DefaultTableExpiration = time.Duration(direct.ValueOf(in.DefaultTableExpirationMs)) * time.Millisecond + out.DefaultEncryptionConfig = EncryptionConfiguration_ToProto(mapCtx, in.DefaultEncryptionConfiguration) out.Description = direct.ValueOf(in.Description) - out.FriendlyName = direct.ValueOf(in.FriendlyName) - out.DatasetReference = DatasetReference_ToAPI(mapCtx, in, name) out.Location = direct.ValueOf(in.Location) - out.IsCaseInsensitive = direct.ValueOf(in.IsCaseInsensitive) if in.MaxTimeTravelHours != nil { - out.MaxTimeTravelHours, _ = strconv.ParseInt(direct.ValueOf(in.MaxTimeTravelHours), 10, 64) + maxHours, _ := strconv.ParseInt(direct.ValueOf(in.MaxTimeTravelHours), 10, 64) + out.MaxTimeTravel = time.Duration(maxHours) * time.Hour } + out.IsCaseInsensitive = direct.ValueOf(in.IsCaseInsensitive) + out.Name = direct.ValueOf(in.FriendlyName) out.StorageBillingModel = direct.ValueOf(in.StorageBillingModel) return out } -func BigQueryDatasetSpec_FromAPI(mapCtx *direct.MapContext, in *api.Dataset) *krm.BigQueryDatasetSpec { +func BigQueryDatasetSpec_FromProto(mapCtx *direct.MapContext, in *pb.DatasetMetadata) *krm.BigQueryDatasetSpec { if in == nil { return nil } out := &krm.BigQueryDatasetSpec{} accessList := []krm.Access{} for _, access := range in.Access { - curAccess := Access_FromAPI(mapCtx, access) + curAccess := AccessEntry_FromProto(mapCtx, access) accessList = append(accessList, direct.ValueOf(curAccess)) } out.Access = accessList out.DefaultCollation = direct.LazyPtr(in.DefaultCollation) - out.DefaultPartitionExpirationMs = direct.LazyPtr(in.DefaultPartitionExpirationMs) - out.DefaultTableExpirationMs = direct.LazyPtr(in.DefaultTableExpirationMs) - out.DefaultEncryptionConfiguration = EncryptionConfiguration_FromAPI(mapCtx, in.DefaultEncryptionConfiguration) + out.DefaultPartitionExpirationMs = direct.LazyPtr(in.DefaultPartitionExpiration.Milliseconds()) + out.DefaultTableExpirationMs = direct.LazyPtr(in.DefaultTableExpiration.Milliseconds()) + out.DefaultEncryptionConfiguration = EncryptionConfiguration_FromProto(mapCtx, in.DefaultEncryptionConfig) out.Description = direct.LazyPtr(in.Description) - out.FriendlyName = direct.LazyPtr(in.FriendlyName) + out.FriendlyName = direct.LazyPtr(in.Name) out.Location = direct.LazyPtr(in.Location) + maxTimeInHours := fmt.Sprintf("%v", in.MaxTimeTravel.Hours()) + out.MaxTimeTravelHours = direct.LazyPtr(maxTimeInHours) out.IsCaseInsensitive = direct.LazyPtr(in.IsCaseInsensitive) - maxTime := strconv.FormatInt(in.MaxTimeTravelHours, 10) - out.MaxTimeTravelHours = direct.LazyPtr(maxTime) out.StorageBillingModel = direct.LazyPtr(in.StorageBillingModel) return out } -func BigQueryDatasetStatus_FromAPI(mapCtx *direct.MapContext, in *api.Dataset) *krm.BigQueryDatasetStatus { +func BigQueryDatasetStatus_FromProto(mapCtx *direct.MapContext, in *pb.DatasetMetadata) *krm.BigQueryDatasetStatus { if in == nil { return nil } out := &krm.BigQueryDatasetStatus{} - out.Etag = direct.LazyPtr(in.Etag) - out.CreationTime = direct.LazyPtr(in.CreationTime) - out.LastModifiedTime = direct.LazyPtr(in.LastModifiedTime) - out.SelfLink = direct.LazyPtr(in.SelfLink) + out.Etag = direct.LazyPtr(in.ETag) + out.CreationTime = direct.LazyPtr(in.CreationTime.UnixMilli()) + out.LastModifiedTime = direct.LazyPtr(in.LastModifiedTime.UnixMilli()) + // The full dataset ID in the form projectID:datasetID + tokens := strings.Split(in.FullID, ":") + if len(tokens) == 2 { + out.SelfLink = direct.LazyPtr(fmt.Sprintf("https://bigquery.googleapis.com/bigquery/v2/projects/%s/datasets/%s", tokens[0], tokens[1])) + } out.ObservedState = &krm.BigQueryDatasetObservedState{Location: direct.LazyPtr(in.Location)} return out } -func BigQueryDatasetStatus_ToAPI(mapCtx *direct.MapContext, in *krm.BigQueryDatasetStatus) *api.Dataset { +func BigQueryDatasetStatus_ToProto(mapCtx *direct.MapContext, in *krm.BigQueryDatasetStatus) *pb.DatasetMetadata { if in == nil { return nil } - out := &api.Dataset{} - out.Etag = direct.ValueOf(in.Etag) - out.CreationTime = direct.ValueOf(in.CreationTime) - out.LastModifiedTime = direct.ValueOf(in.LastModifiedTime) - out.SelfLink = direct.ValueOf(in.SelfLink) - out.Location = direct.ValueOf(in.ObservedState.Location) + out := &pb.DatasetMetadata{} + out.ETag = direct.ValueOf(in.Etag) + out.CreationTime = direct.UnixMillisToTime(direct.ValueOf(in.CreationTime)) + out.LastModifiedTime = direct.UnixMillisToTime(direct.ValueOf(in.LastModifiedTime)) + // The full dataset ID in the form projectID:datasetID + if in.SelfLink != nil { + selfLink := strings.Trim(direct.ValueOf(in.SelfLink), "https://bigquery.googleapis.com/bigquery/v2/") + tokens := strings.Split(selfLink, "/") + if len(tokens) == 4 && tokens[0] == "projects" && tokens[2] == "datasets" { + out.FullID = fmt.Sprintf("%s:%s", tokens[1], tokens[3]) + } + } return out } -func Access_ToAPI(mapCtx *direct.MapContext, in *krm.Access) *api.DatasetAccess { +func AccessEntry_ToProto(mapCtx *direct.MapContext, in *krm.Access) *pb.AccessEntry { if in == nil { return nil } - out := &api.DatasetAccess{} - out.Domain = direct.ValueOf(in.Domain) - out.GroupByEmail = direct.ValueOf(in.GroupByEmail) - out.IamMember = direct.ValueOf(in.IamMember) - out.UserByEmail = direct.ValueOf(in.UserByEmail) - out.SpecialGroup = direct.ValueOf(in.SpecialGroup) - out.Role = direct.ValueOf(in.Role) - out.Dataset = DatasetAccessEntry_ToAPI(mapCtx, in.Dataset) - out.Routine = RoutineReference_ToAPI(mapCtx, in.Routine) - out.View = TableReference_ToAPI(mapCtx, in.View) + out := &pb.AccessEntry{} + out.Role = pb.AccessRole(direct.ValueOf(in.Role)) + switch { + case in.Domain != nil: + out.Entity = direct.ValueOf(in.Domain) + out.EntityType = pb.DomainEntity + case in.GroupByEmail != nil: + out.Entity = direct.ValueOf(in.GroupByEmail) + out.EntityType = pb.GroupEmailEntity + case in.UserByEmail != nil: + out.Entity = direct.ValueOf(in.UserByEmail) + out.EntityType = pb.UserEmailEntity + case in.SpecialGroup != nil: + out.Entity = direct.ValueOf(in.SpecialGroup) + out.EntityType = pb.SpecialGroupEntity + case in.View != nil: + out.View = TableReference_ToProto(mapCtx, in.View) + out.EntityType = pb.ViewEntity + case in.IamMember != nil: + out.Entity = direct.ValueOf(in.IamMember) + out.EntityType = pb.IAMMemberEntity + case in.Routine != nil: + out.Routine = RoutineReference_ToProto(mapCtx, in.Routine) + out.EntityType = pb.ViewEntity + case in.Dataset != nil: + out.Dataset = DatasetAccessEntry_ToProto(mapCtx, in.Dataset) + out.EntityType = pb.ViewEntity + } return out } -func Access_FromAPI(mapCtx *direct.MapContext, in *api.DatasetAccess) *krm.Access { +func AccessEntry_FromProto(mapCtx *direct.MapContext, in *pb.AccessEntry) *krm.Access { if in == nil { return nil } out := &krm.Access{} - out.Domain = direct.LazyPtr(in.Domain) - out.GroupByEmail = direct.LazyPtr(in.GroupByEmail) - out.IamMember = direct.LazyPtr(in.IamMember) - out.UserByEmail = direct.LazyPtr(in.UserByEmail) - out.SpecialGroup = direct.LazyPtr(in.SpecialGroup) - out.Role = direct.LazyPtr(in.Role) - out.Dataset = DatasetAccessEntry_FromAPI(mapCtx, in.Dataset) - out.Routine = RoutineReference_FromAPI(mapCtx, in.Routine) - out.View = TableReference_FromAPI(mapCtx, in.View) + out.Role = direct.LazyPtr(string(in.Role)) + switch in.EntityType { + case pb.DomainEntity: + out.Domain = direct.LazyPtr(in.Entity) + case pb.GroupEmailEntity: + out.GroupByEmail = direct.LazyPtr(in.Entity) + case pb.UserEmailEntity: + out.UserByEmail = direct.LazyPtr(in.Entity) + case pb.SpecialGroupEntity: + out.SpecialGroup = direct.LazyPtr(in.Entity) + case pb.ViewEntity: + out.View = TableReference_FromProto(mapCtx, in.View) + case pb.IAMMemberEntity: + out.IamMember = direct.LazyPtr(in.Entity) + case pb.RoutineEntity: + out.Routine = RoutineReference_FromProto(mapCtx, in.Routine) + case pb.DatasetEntity: + out.Dataset = DatasetAccessEntry_FromProto(mapCtx, in.Dataset) + } + return out +} +func BigQueryDataset_ToMetadataToUpdate(mapCtx *direct.MapContext, in *pb.DatasetMetadata, updatePaths []string) *pb.DatasetMetadataToUpdate { + if in == nil { + return nil + } + out := &pb.DatasetMetadataToUpdate{} + acccessList := []*pb.AccessEntry{} + for _, access := range in.Access { + acccessList = append(acccessList, access) + } + out.Access = acccessList + if in.DefaultEncryptionConfig != nil { + out.DefaultEncryptionConfig = &pb.EncryptionConfig{ + KMSKeyName: in.DefaultEncryptionConfig.KMSKeyName, + } + } + // if the value to explicitly set to empty in the update request, we set the value. + // Otherwise, we drop the value. + if in.DefaultCollation != "" || slices.Contains(updatePaths, "default_collation") { + out.DefaultCollation = in.DefaultCollation + } + if in.DefaultPartitionExpiration != 0 || slices.Contains(updatePaths, "default_partition_expiration") { + out.DefaultPartitionExpiration = in.DefaultPartitionExpiration + } + if in.DefaultTableExpiration != 0 || slices.Contains(updatePaths, "default_table_expiration") { + out.DefaultTableExpiration = in.DefaultTableExpiration + } + if in.Description != "" || slices.Contains(updatePaths, "description") { + out.Description = in.Description + } + if in.MaxTimeTravel != 0 || slices.Contains(updatePaths, "max_time_travel") { + out.MaxTimeTravel = in.MaxTimeTravel + } + out.IsCaseInsensitive = in.IsCaseInsensitive + if in.Name != "" || slices.Contains(updatePaths, "friendly_name") { + out.Name = in.Name + } + if in.StorageBillingModel != "" || slices.Contains(updatePaths, "storage_billing_model") { + out.StorageBillingModel = in.StorageBillingModel + } return out } -func DatasetAccessEntry_FromAPI(mapCtx *direct.MapContext, in *api.DatasetAccessEntry) *krm.DatasetAccessEntry { +func DatasetAccessEntry_FromProto(mapCtx *direct.MapContext, in *pb.DatasetAccessEntry) *krm.DatasetAccessEntry { if in == nil { return nil } out := &krm.DatasetAccessEntry{} - out.Dataset = &krm.DatasetReference{ - DatasetId: direct.LazyPtr(in.Dataset.DatasetId), - ProjectId: direct.LazyPtr(in.Dataset.ProjectId), + out.Dataset = DatasetReference_FromProto(mapCtx, in.Dataset) + for _, targetType := range in.TargetTypes { + out.TargetTypes = append(out.TargetTypes, targetType) } - out.TargetTypes = in.TargetTypes return out } -func DatasetAccessEntry_ToAPI(mapCtx *direct.MapContext, in *krm.DatasetAccessEntry) *api.DatasetAccessEntry { +func DatasetAccessEntry_ToProto(mapCtx *direct.MapContext, in *krm.DatasetAccessEntry) *pb.DatasetAccessEntry { if in == nil { return nil } - out := &api.DatasetAccessEntry{} - out.Dataset = &api.DatasetReference{ - DatasetId: direct.ValueOf(in.Dataset.DatasetId), - ProjectId: direct.ValueOf(in.Dataset.ProjectId), + out := &pb.DatasetAccessEntry{} + out.Dataset = DatasetReference_ToProto(mapCtx, in.Dataset) + out.TargetTypes = make([]string, len(in.TargetTypes)) + for _, targetType := range in.TargetTypes { + out.TargetTypes = append(out.TargetTypes, targetType) } - out.TargetTypes = in.TargetTypes return out } -func DatasetReference_ToAPI(mapCtx *direct.MapContext, in *krm.BigQueryDatasetSpec, name string) *api.DatasetReference { +func DatasetReference_FromProto(mapCtx *direct.MapContext, in *pb.Dataset) *krm.DatasetReference { if in == nil { return nil } - out := &api.DatasetReference{} - out.DatasetId = name + out := &krm.DatasetReference{} + out.DatasetId = direct.LazyPtr(in.DatasetID) + out.ProjectId = direct.LazyPtr(in.ProjectID) return out } -func EncryptionConfiguration_ToAPI(mapCtx *direct.MapContext, in *krm.EncryptionConfiguration) *api.EncryptionConfiguration { +func DatasetReference_ToProto(mapCtx *direct.MapContext, in *krm.DatasetReference) *pb.Dataset { + if in == nil { + return nil + } + out := &pb.Dataset{} + out.DatasetID = direct.ValueOf(in.DatasetId) + out.ProjectID = direct.ValueOf(in.ProjectId) + return out +} +func DatasetSpec_ToExternalDatasetReference(mapCtx *direct.MapContext, in *krm.BigQueryDatasetSpec) *pb.ExternalDatasetReference { + // **NOTYET** + // There are no matching fields in KRM for now. + // out.Connection + // out.ExternalSource + return nil +} +func EncryptionConfiguration_ToProto(mapCtx *direct.MapContext, in *krm.EncryptionConfiguration) *pb.EncryptionConfig { if in == nil { return nil } - out := &api.EncryptionConfiguration{} + out := &pb.EncryptionConfig{} if in.KmsKeyRef != nil { - out.KmsKeyName = in.KmsKeyRef.External + out.KMSKeyName = in.KmsKeyRef.External } return out } -func EncryptionConfiguration_FromAPI(mapCtx *direct.MapContext, in *api.EncryptionConfiguration) *krm.EncryptionConfiguration { +func EncryptionConfiguration_FromProto(mapCtx *direct.MapContext, in *pb.EncryptionConfig) *krm.EncryptionConfiguration { if in == nil { return nil } out := &krm.EncryptionConfiguration{} out.KmsKeyRef = &v1beta1.KMSCryptoKeyRef{ - External: in.KmsKeyName, + External: in.KMSKeyName, } return out } -func RoutineReference_FromAPI(mapCtx *direct.MapContext, in *api.RoutineReference) *krm.RoutineReference { +func RoutineReference_FromProto(mapCtx *direct.MapContext, in *pb.Routine) *krm.RoutineReference { if in == nil { return nil } out := &krm.RoutineReference{} - out.DatasetId = direct.LazyPtr(in.DatasetId) - out.ProjectId = direct.LazyPtr(in.ProjectId) - out.RoutineId = direct.LazyPtr(in.RoutineId) + out.DatasetId = direct.LazyPtr(in.DatasetID) + out.ProjectId = direct.LazyPtr(in.ProjectID) + out.RoutineId = direct.LazyPtr(in.RoutineID) return out } -func RoutineReference_ToAPI(mapCtx *direct.MapContext, in *krm.RoutineReference) *api.RoutineReference { +func RoutineReference_ToProto(mapCtx *direct.MapContext, in *krm.RoutineReference) *pb.Routine { if in == nil { return nil } - out := &api.RoutineReference{} - out.DatasetId = direct.ValueOf(in.DatasetId) - out.ProjectId = direct.ValueOf(in.ProjectId) - out.RoutineId = direct.ValueOf(in.RoutineId) + out := &pb.Routine{} + out.DatasetID = direct.ValueOf(in.DatasetId) + out.ProjectID = direct.ValueOf(in.ProjectId) + out.RoutineID = direct.ValueOf(in.RoutineId) return out } -func TableReference_FromAPI(mapCtx *direct.MapContext, in *api.TableReference) *krm.TableReference { +func TableReference_FromProto(mapCtx *direct.MapContext, in *pb.Table) *krm.TableReference { if in == nil { return nil } out := &krm.TableReference{} - out.DatasetId = direct.LazyPtr(in.DatasetId) - out.ProjectId = direct.LazyPtr(in.ProjectId) - out.TableId = direct.LazyPtr(in.TableId) + out.DatasetId = direct.LazyPtr(in.DatasetID) + out.ProjectId = direct.LazyPtr(in.ProjectID) + out.TableId = direct.LazyPtr(in.TableID) return out } -func TableReference_ToAPI(mapCtx *direct.MapContext, in *krm.TableReference) *api.TableReference { +func TableReference_ToProto(mapCtx *direct.MapContext, in *krm.TableReference) *pb.Table { if in == nil { return nil } - out := &api.TableReference{} - out.DatasetId = direct.ValueOf(in.DatasetId) - out.ProjectId = direct.ValueOf(in.ProjectId) - out.TableId = direct.ValueOf(in.TableId) + out := &pb.Table{} + out.DatasetID = direct.ValueOf(in.DatasetId) + out.ProjectID = direct.ValueOf(in.ProjectId) + out.TableID = direct.ValueOf(in.TableId) return out } diff --git a/pkg/controller/direct/bigquerydataset/dataset_controller.go b/pkg/controller/direct/bigquerydataset/dataset_controller.go index b2f837d148..61b4b79047 100644 --- a/pkg/controller/direct/bigquerydataset/dataset_controller.go +++ b/pkg/controller/direct/bigquerydataset/dataset_controller.go @@ -26,8 +26,7 @@ import ( "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/direct/directbase" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/direct/registry" - clone "github.com/huandu/go-clone" - api "google.golang.org/api/bigquery/v2" + bigquery "cloud.google.com/go/bigquery" "google.golang.org/api/option" "google.golang.org/protobuf/types/known/fieldmaskpb" @@ -56,13 +55,13 @@ type model struct { config config.ControllerConfig } -func (m *model) service(ctx context.Context) (*api.Service, error) { +func (m *model) service(ctx context.Context, projectID string) (*bigquery.Client, error) { var opts []option.ClientOption opts, err := m.config.RESTClientOptions() if err != nil { return nil, err } - gcpService, err := api.NewService(ctx, opts...) + gcpService, err := bigquery.NewClient(ctx, projectID, opts...) if err != nil { return nil, fmt.Errorf("building Dataset client: %w", err) } @@ -74,22 +73,32 @@ func (m *model) AdapterForObject(ctx context.Context, reader client.Reader, u *u if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, &obj); err != nil { return nil, fmt.Errorf("error converting to %T: %w", obj, err) } - + hasUpdateIsCaseInsensitive := false + if obj.Spec.IsCaseInsensitive != nil { + hasUpdateIsCaseInsensitive = true + } id, err := krm.NewBigQueryDatasetRef(ctx, reader, obj) if err != nil { return nil, err } + parent, _, err := krm.ParseBigQueryDatasetExternal(id.External) + projectID := parent.ProjectID + if projectID == "" { + return nil, fmt.Errorf("cannot resolve project ID") + } + // Get bigquery GCP client - gcpService, err := m.service(ctx) + gcpService, err := m.service(ctx, projectID) if err != nil { return nil, err } return &Adapter{ - id: id, - gcpService: gcpService, - desired: obj, - reader: reader, + id: id, + gcpService: gcpService, + desired: obj, + reader: reader, + hasUpdateIsCaseInsensitive: hasUpdateIsCaseInsensitive, }, nil } @@ -99,11 +108,12 @@ func (m *model) AdapterForURL(ctx context.Context, url string) (directbase.Adapt } type Adapter struct { - id *krm.BigQueryDatasetRef - gcpService *api.Service - desired *krm.BigQueryDataset - actual *api.Dataset - reader client.Reader + id *krm.BigQueryDatasetRef + gcpService *bigquery.Client + desired *krm.BigQueryDataset + actual *bigquery.DatasetMetadata + reader client.Reader + hasUpdateIsCaseInsensitive bool } var _ directbase.Adapter = &Adapter{} @@ -116,8 +126,8 @@ func (a *Adapter) Find(ctx context.Context) (bool, error) { if err != nil { return false, fmt.Errorf("failed to parse bigquery dataset full name, %w", err) } - datasetGetCall := a.gcpService.Datasets.Get(parent.ProjectID, datasetId) - datasetpb, err := datasetGetCall.Do() + dsHandler := a.gcpService.DatasetInProject(parent.ProjectID, datasetId) + datasetpb, err := dsHandler.Metadata(ctx) if err != nil { if direct.IsNotFound(err) { return false, nil @@ -134,13 +144,13 @@ func (a *Adapter) Create(ctx context.Context, createOp *directbase.CreateOperati log.V(2).Info("creating Dataset", "name", a.id.External) mapCtx := &direct.MapContext{} - desiredDataset := BigQueryDatasetSpec_ToAPI(mapCtx, &a.desired.Spec, a.desired.Name) + desiredDataset := BigQueryDatasetSpec_ToProto(mapCtx, &a.desired.Spec) desiredDataset.Labels = make(map[string]string) for k, v := range a.desired.GetObjectMeta().GetLabels() { desiredDataset.Labels[k] = v } desiredDataset.Labels["managed-by-cnrm"] = "true" - parent, _, err := krm.ParseBigQueryDatasetExternal(a.id.External) + parent, datasetId, err := krm.ParseBigQueryDatasetExternal(a.id.External) if err != nil { return fmt.Errorf("failed to parse bigquery dataset full name, %w", err) } @@ -150,17 +160,25 @@ func (a *Adapter) Create(ctx context.Context, createOp *directbase.CreateOperati if err != nil { return err } - desiredDataset.DefaultEncryptionConfiguration.KmsKeyName = kmsRef.External + desiredDataset.DefaultEncryptionConfig.KMSKeyName = kmsRef.External } - insertDatasetCall := a.gcpService.Datasets.Insert(parent.ProjectID, desiredDataset) - inserted, err := insertDatasetCall.Do() - if err != nil { - return fmt.Errorf("inserting Dataset %s: %w", a.id.External, err) + dsHandler := a.gcpService.DatasetInProject(parent.ProjectID, datasetId) + if err := dsHandler.Create(ctx, desiredDataset); err != nil { + return fmt.Errorf("Error creating Dataset %s: %w", a.id.External, err) } - log.V(2).Info("successfully inserted Dataset", "name", a.id.External) + log.V(2).Info("successfully created Dataset", "name", a.id.External) + // The bigquery go client Create() does not return the created dataset. + // Fetching the dataset metadata + createdMetadata, err := dsHandler.Metadata(ctx) + if err != nil { + if direct.IsNotFound(err) { + return nil + } + return fmt.Errorf("Error getting the created BigQueryDataset %q: %w", a.id.External, err) + } status := &krm.BigQueryDatasetStatus{} - status = BigQueryDatasetStatus_FromAPI(mapCtx, inserted) + status = BigQueryDatasetStatus_FromProto(mapCtx, createdMetadata) if mapCtx.Err() != nil { return mapCtx.Err() } @@ -177,91 +195,96 @@ func (a *Adapter) Update(ctx context.Context, updateOp *directbase.UpdateOperati // Convert KRM object to proto message desiredKRM := a.desired.DeepCopy() - desired := BigQueryDatasetSpec_ToAPI(mapCtx, &desiredKRM.Spec, desiredKRM.Name) + desired := BigQueryDatasetSpec_ToProto(mapCtx, &desiredKRM.Spec) if mapCtx.Err() != nil { return mapCtx.Err() } - resource := clone.Clone(a.actual).(*api.Dataset) - + resource := cloneBigQueryDatasetMetadate(a.actual) // Check for immutable fields - if desired.Location != "" && !reflect.DeepEqual(desired.Location, resource.Location) { + if desiredKRM.Spec.Location != nil && !reflect.DeepEqual(desired.Location, resource.Location) { return fmt.Errorf("BigQueryDataset %s/%s location cannot be changed, actual: %s, desired: %s", u.GetNamespace(), u.GetName(), resource.Location, desired.Location) } - // Find diff updateMask := &fieldmaskpb.FieldMask{} - if !reflect.DeepEqual(desired.Description, resource.Description) { + if desired.Description != "" && !reflect.DeepEqual(desired.Description, resource.Description) { resource.Description = desired.Description updateMask.Paths = append(updateMask.Paths, "description") } - if !reflect.DeepEqual(desired.FriendlyName, resource.FriendlyName) { - resource.FriendlyName = desired.FriendlyName + if desired.Name != "" && !reflect.DeepEqual(desired.Name, resource.Name) { + resource.Name = desired.Name updateMask.Paths = append(updateMask.Paths, "friendly_name") } - if !reflect.DeepEqual(desired.DefaultPartitionExpirationMs, resource.DefaultPartitionExpirationMs) { - resource.DefaultPartitionExpirationMs = desired.DefaultPartitionExpirationMs - updateMask.Paths = append(updateMask.Paths, "default_partition_expirationMs") + if desired.DefaultPartitionExpiration != 0 && !reflect.DeepEqual(desired.DefaultPartitionExpiration, resource.DefaultPartitionExpiration) { + resource.DefaultPartitionExpiration = desired.DefaultPartitionExpiration + updateMask.Paths = append(updateMask.Paths, "default_partition_expiration") } - if !reflect.DeepEqual(desired.DefaultTableExpirationMs, resource.DefaultTableExpirationMs) { - resource.DefaultTableExpirationMs = desired.DefaultTableExpirationMs - updateMask.Paths = append(updateMask.Paths, "default_table_expirationMs") + if desired.DefaultTableExpiration != 0 && !reflect.DeepEqual(desired.DefaultTableExpiration, resource.DefaultTableExpiration) { + resource.DefaultTableExpiration = desired.DefaultTableExpiration + updateMask.Paths = append(updateMask.Paths, "default_table_expiration") } - if !reflect.DeepEqual(desired.DefaultCollation, resource.DefaultCollation) { + if desired.DefaultCollation != "" && !reflect.DeepEqual(desired.DefaultCollation, resource.DefaultCollation) { resource.DefaultCollation = desired.DefaultCollation updateMask.Paths = append(updateMask.Paths, "default_collation") } - if desired.DefaultEncryptionConfiguration != nil && resource.DefaultEncryptionConfiguration != nil && !reflect.DeepEqual(desired.DefaultEncryptionConfiguration, resource.DefaultEncryptionConfiguration) { + if desired.DefaultEncryptionConfig != nil && resource.DefaultEncryptionConfig != nil && !reflect.DeepEqual(desired.DefaultEncryptionConfig, resource.DefaultEncryptionConfig) { // Resolve KMS key reference if a.desired.Spec.DefaultEncryptionConfiguration != nil { kmsRef, err := refs.ResolveKMSCryptoKeyRef(ctx, a.reader, a.desired, a.desired.Spec.DefaultEncryptionConfiguration.KmsKeyRef) if err != nil { return err } - desired.DefaultEncryptionConfiguration.KmsKeyName = kmsRef.External + desired.DefaultEncryptionConfig.KMSKeyName = kmsRef.External } - resource.DefaultEncryptionConfiguration.KmsKeyName = desired.DefaultEncryptionConfiguration.KmsKeyName + resource.DefaultEncryptionConfig.KMSKeyName = desired.DefaultEncryptionConfig.KMSKeyName updateMask.Paths = append(updateMask.Paths, "default_encryption_configuration") } - if !reflect.DeepEqual(desired.IsCaseInsensitive, resource.IsCaseInsensitive) { + if a.hasUpdateIsCaseInsensitive && !reflect.DeepEqual(desired.IsCaseInsensitive, resource.IsCaseInsensitive) { resource.IsCaseInsensitive = desired.IsCaseInsensitive updateMask.Paths = append(updateMask.Paths, "is_case_sensitive") } - if !reflect.DeepEqual(desired.MaxTimeTravelHours, resource.MaxTimeTravelHours) { - resource.MaxTimeTravelHours = desired.MaxTimeTravelHours - updateMask.Paths = append(updateMask.Paths, "max_time_interval_hours") + if desired.StorageBillingModel != "" && !reflect.DeepEqual(desired.StorageBillingModel, resource.StorageBillingModel) { + resource.StorageBillingModel = desired.StorageBillingModel + updateMask.Paths = append(updateMask.Paths, "storage_billing_model") + } + // If we do not set a value, the GCP service defaults to 168 + // If the existing value is 168, it means that we did not set this field at creation and it defaults to 168. + // So if the desired value is 0, it means that we do not intend to update this field. + if desired.MaxTimeTravel != 0 && !reflect.DeepEqual(desired.MaxTimeTravel, resource.MaxTimeTravel) && (resource.MaxTimeTravel != 168 && desired.MaxTimeTravel != 0) { + resource.MaxTimeTravel = desired.MaxTimeTravel + updateMask.Paths = append(updateMask.Paths, "max_time_travel") } if desired.Access != nil && resource.Access != nil && len(desired.Access) > 0 && !reflect.DeepEqual(desired.Access, resource.Access) { for _, access := range desired.Access { resource.Access = append(resource.Access, access) } - updateMask.Paths = append(updateMask.Paths, "access") } - if !reflect.DeepEqual(desired.StorageBillingModel, resource.StorageBillingModel) { - resource.StorageBillingModel = desired.StorageBillingModel - updateMask.Paths = append(updateMask.Paths, "storage_billing_model") - } - if len(updateMask.Paths) == 0 { return nil } + + // Get parent information parent, datasetId, err := krm.ParseBigQueryDatasetExternal(a.id.External) if err != nil { return fmt.Errorf("failed to parse bigquery dataset full name, %w", err) } - if desired.Access == nil || len(desired.Access) == 0 { - resource.Access = a.actual.Access + // Compute the dataset metadate for update request + datasetMetadataToUpdate := BigQueryDataset_ToMetadataToUpdate(mapCtx, resource, updateMask.Paths) + for k, v := range a.desired.GetObjectMeta().GetLabels() { + datasetMetadataToUpdate.SetLabel(k, v) } - updateDatasetCall := a.gcpService.Datasets.Update(parent.ProjectID, datasetId, resource) - updated, err := updateDatasetCall.Do() + datasetMetadataToUpdate.SetLabel("managed-by-cnrm", "true") + // Call update + dsHandler := a.gcpService.DatasetInProject(parent.ProjectID, datasetId) + updated, err := dsHandler.Update(ctx, *datasetMetadataToUpdate, resource.ETag) if err != nil { return fmt.Errorf("updating Dataset %s: %w", a.id.External, err) } log.V(2).Info("successfully updated Dataset", "name", a.id.External) status := &krm.BigQueryDatasetStatus{} - status = BigQueryDatasetStatus_FromAPI(mapCtx, updated) + status = BigQueryDatasetStatus_FromProto(mapCtx, updated) if mapCtx.Err() != nil { return mapCtx.Err() } @@ -276,7 +299,7 @@ func (a *Adapter) Export(ctx context.Context) (*unstructured.Unstructured, error obj := &krm.BigQueryDataset{} mapCtx := &direct.MapContext{} - obj.Spec = direct.ValueOf(BigQueryDatasetSpec_FromAPI(mapCtx, a.actual)) + obj.Spec = direct.ValueOf(BigQueryDatasetSpec_FromProto(mapCtx, a.actual)) if mapCtx.Err() != nil { return nil, mapCtx.Err() } @@ -303,9 +326,8 @@ func (a *Adapter) Delete(ctx context.Context, deleteOp *directbase.DeleteOperati if err != nil { return false, fmt.Errorf("failed to parse bigquery dataset full name, %w", err) } - deleteDatasetCall := a.gcpService.Datasets.Delete(parent.ProjectID, datasetId) - err = deleteDatasetCall.Do() - if err != nil { + dsHandler := a.gcpService.DatasetInProject(parent.ProjectID, datasetId) + if err := dsHandler.DeleteWithContents(ctx); err != nil { return false, fmt.Errorf("deleting Dataset %s: %w", a.id.External, err) } log.V(2).Info("successfully deleted Dataset", "name", a.id.External) diff --git a/pkg/controller/direct/bigquerydataset/utils.go b/pkg/controller/direct/bigquerydataset/utils.go index 71e4335e1e..c960c771b9 100644 --- a/pkg/controller/direct/bigquerydataset/utils.go +++ b/pkg/controller/direct/bigquerydataset/utils.go @@ -18,6 +18,7 @@ import ( "encoding/json" "fmt" + bigquery "cloud.google.com/go/bigquery" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/reflect/protoreflect" ) @@ -55,3 +56,49 @@ func convertAPIToProto[V protoreflect.ProtoMessage](u any, pV *V) error { *pV = v return nil } +func cloneBigQueryDatasetMetadate(in *bigquery.DatasetMetadata) *bigquery.DatasetMetadata { + if in == nil { + return nil + } + out := &bigquery.DatasetMetadata{} + acccessList := []*bigquery.AccessEntry{} + for _, access := range in.Access { + acccessList = append(acccessList, access) + } + out.Access = acccessList + if in.DefaultEncryptionConfig != nil { + out.DefaultEncryptionConfig = &bigquery.EncryptionConfig{ + KMSKeyName: in.DefaultEncryptionConfig.KMSKeyName, + } + } + out.Location = in.Location + // if the value to explicitly set to empty in the update request, we set the value. + // Otherwise, we drop the value. + if in.DefaultCollation != "" { + out.DefaultCollation = in.DefaultCollation + } + if in.DefaultPartitionExpiration != 0 { + out.DefaultPartitionExpiration = in.DefaultPartitionExpiration + } + if in.DefaultTableExpiration != 0 { + out.DefaultTableExpiration = in.DefaultTableExpiration + } + if in.Description != "" { + out.Description = in.Description + } + if in.MaxTimeTravel != 0 { + out.MaxTimeTravel = in.MaxTimeTravel + } + out.IsCaseInsensitive = in.IsCaseInsensitive + if in.Name != "" { + out.Name = in.Name + } + if in.StorageBillingModel != "" { + out.StorageBillingModel = in.StorageBillingModel + } + out.CreationTime = in.CreationTime + out.LastModifiedTime = in.LastModifiedTime + out.ETag = in.ETag + out.FullID = in.FullID + return out +} diff --git a/pkg/controller/direct/maputils.go b/pkg/controller/direct/maputils.go index 34ba5a03f5..fc09883996 100644 --- a/pkg/controller/direct/maputils.go +++ b/pkg/controller/direct/maputils.go @@ -334,3 +334,13 @@ func Int64Value_FromProto(mapCtx *MapContext, ts *wrapperspb.Int64Value) int64 { func Int64Value_ToProto(mapCtx *MapContext, s int64) *wrapperspb.Int64Value { return wrapperspb.Int64(s) } + +// Convert a number of milliseconds since the Unix epoch to a time.Time. +// Treat an input of zero specially: convert it to the zero time, +// rather than the start of the epoch. +func UnixMillisToTime(m int64) time.Time { + if m == 0 { + return time.Time{} + } + return time.Unix(0, m*1e6) +} diff --git a/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/basicbigquerydataset-direct/_http.log b/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/basicbigquerydataset-direct/_http.log index bd2610d99f..c0d98ced21 100644 --- a/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/basicbigquerydataset-direct/_http.log +++ b/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/basicbigquerydataset-direct/_http.log @@ -150,8 +150,9 @@ X-Xss-Protection: 0 --- -PUT https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&prettyPrint=false +PATCH https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&prettyPrint=false Content-Type: application/json +If-Match: abcdef0123A= User-Agent: kcc/controller-manager { @@ -173,23 +174,13 @@ User-Agent: kcc/controller-manager "specialGroup": "projectWriters" } ], - "creationTime": "123456789", - "datasetReference": { - "datasetId": "bigquerydatasetsample${uniqueId}", - "projectId": "${projectId}" - }, - "etag": "abcdef0123A=", "friendlyName": "bigquerydataset-sample-updated", - "id": "000000000000000000000", - "kind": "bigquery#dataset", + "isCaseInsensitive": false, "labels": { "cnrm-test": "true", "managed-by-cnrm": "true" }, - "lastModifiedTime": "123456789", - "location": "US", - "selfLink": "https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/bigquerydatasetsample${uniqueId}", - "type": "DEFAULT" + "maxTimeTravelHours": "168" } 200 OK @@ -230,6 +221,7 @@ X-Xss-Protection: 0 "etag": "abcdef0123A=", "friendlyName": "bigquerydataset-sample-updated", "id": "000000000000000000000", + "isCaseInsensitive": false, "kind": "bigquery#dataset", "labels": { "cnrm-test": "true", @@ -237,6 +229,7 @@ X-Xss-Protection: 0 }, "lastModifiedTime": "123456789", "location": "US", + "maxTimeTravelHours": "168", "selfLink": "https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/bigquerydatasetsample${uniqueId}", "type": "DEFAULT" } @@ -285,6 +278,7 @@ X-Xss-Protection: 0 "etag": "abcdef0123A=", "friendlyName": "bigquerydataset-sample-updated", "id": "000000000000000000000", + "isCaseInsensitive": false, "kind": "bigquery#dataset", "labels": { "cnrm-test": "true", @@ -340,6 +334,7 @@ X-Xss-Protection: 0 "etag": "abcdef0123A=", "friendlyName": "bigquerydataset-sample-updated", "id": "000000000000000000000", + "isCaseInsensitive": false, "kind": "bigquery#dataset", "labels": { "cnrm-test": "true", @@ -354,7 +349,7 @@ X-Xss-Protection: 0 --- -DELETE https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&prettyPrint=false +DELETE https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&deleteContents=true&prettyPrint=false User-Agent: kcc/controller-manager 204 No Content diff --git a/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/bigquerydatasetaccessblock-direct/_http.log b/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/bigquerydatasetaccessblock-direct/_http.log index 6dc0ad119f..9df9e3157b 100644 --- a/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/bigquerydatasetaccessblock-direct/_http.log +++ b/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/bigquerydatasetaccessblock-direct/_http.log @@ -151,8 +151,9 @@ X-Xss-Protection: 0 --- -PUT https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&prettyPrint=false +PATCH https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&prettyPrint=false Content-Type: application/json +If-Match: abcdef0123A= User-Agent: kcc/controller-manager { @@ -178,25 +179,15 @@ User-Agent: kcc/controller-manager "specialGroup": "projectOwners" } ], - "creationTime": "123456789", - "datasetReference": { - "datasetId": "bigquerydataset${uniqueId}", - "projectId": "${projectId}" - }, "defaultTableExpirationMs": "7200000", "description": "BigQuery Dataset With Access Block v2", - "etag": "abcdef0123A=", "friendlyName": "bigquerydataset-accessblock", - "id": "000000000000000000000", - "kind": "bigquery#dataset", + "isCaseInsensitive": false, "labels": { "cnrm-test": "true", "managed-by-cnrm": "true" }, - "lastModifiedTime": "123456789", - "location": "US", - "selfLink": "https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/bigquerydataset${uniqueId}", - "type": "DEFAULT" + "maxTimeTravelHours": "168" } 200 OK @@ -243,6 +234,7 @@ X-Xss-Protection: 0 "etag": "abcdef0123A=", "friendlyName": "bigquerydataset-accessblock", "id": "000000000000000000000", + "isCaseInsensitive": false, "kind": "bigquery#dataset", "labels": { "cnrm-test": "true", @@ -250,6 +242,7 @@ X-Xss-Protection: 0 }, "lastModifiedTime": "123456789", "location": "US", + "maxTimeTravelHours": "168", "selfLink": "https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/bigquerydataset${uniqueId}", "type": "DEFAULT" } @@ -304,6 +297,7 @@ X-Xss-Protection: 0 "etag": "abcdef0123A=", "friendlyName": "bigquerydataset-accessblock", "id": "000000000000000000000", + "isCaseInsensitive": false, "kind": "bigquery#dataset", "labels": { "cnrm-test": "true", @@ -365,6 +359,7 @@ X-Xss-Protection: 0 "etag": "abcdef0123A=", "friendlyName": "bigquerydataset-accessblock", "id": "000000000000000000000", + "isCaseInsensitive": false, "kind": "bigquery#dataset", "labels": { "cnrm-test": "true", @@ -379,7 +374,7 @@ X-Xss-Protection: 0 --- -DELETE https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&prettyPrint=false +DELETE https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&deleteContents=true&prettyPrint=false User-Agent: kcc/controller-manager 204 No Content diff --git a/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/fullybigquerydataset-direct/_generated_export_fullybigquerydataset-direct.golden b/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/fullybigquerydataset-direct/_generated_export_fullybigquerydataset-direct.golden index 0523cbdbdf..74a9990584 100644 --- a/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/fullybigquerydataset-direct/_generated_export_fullybigquerydataset-direct.golden +++ b/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/fullybigquerydataset-direct/_generated_export_fullybigquerydataset-direct.golden @@ -15,6 +15,7 @@ spec: specialGroup: projectOwners - role: OWNER userByEmail: user@google.com + defaultCollation: und:ci defaultEncryptionConfiguration: kmsKeyRef: external: projects/${projectId}/locations/us/keyRings/kmskeyring-${uniqueId}/cryptoKeys/kmscryptokey-${uniqueId} diff --git a/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/fullybigquerydataset-direct/_http.log b/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/fullybigquerydataset-direct/_http.log index a43788c8ec..27c5843dae 100644 --- a/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/fullybigquerydataset-direct/_http.log +++ b/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/fullybigquerydataset-direct/_http.log @@ -554,8 +554,9 @@ X-Xss-Protection: 0 --- -PUT https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&prettyPrint=false +PATCH https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&prettyPrint=false Content-Type: application/json +If-Match: abcdef0123A= User-Agent: kcc/controller-manager { @@ -577,31 +578,21 @@ User-Agent: kcc/controller-manager "userByEmail": "user@google.com" } ], - "creationTime": "123456789", - "datasetReference": { - "datasetId": "bigquerydataset${uniqueId}", - "projectId": "${projectId}" - }, + "defaultCollation": "und:ci", "defaultEncryptionConfiguration": { "kmsKeyName": "projects/${projectId}/locations/us/keyRings/kmskeyring-${uniqueId}/cryptoKeys/kmscryptokey-${uniqueId}" }, "defaultPartitionExpirationMs": "3800000", "defaultTableExpirationMs": "3800000", "description": "Fully Configured BigQuery Dataset updated", - "etag": "abcdef0123A=", "friendlyName": "bigquerydataset-fullyconfigured-updated", - "id": "000000000000000000000", - "kind": "bigquery#dataset", + "isCaseInsensitive": false, "labels": { "cnrm-test": "true", "managed-by-cnrm": "true" }, - "lastModifiedTime": "123456789", - "location": "US", "maxTimeTravelHours": "96", - "selfLink": "https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/bigquerydataset${uniqueId}", - "storageBillingModel": "LOGICAL", - "type": "DEFAULT" + "storageBillingModel": "LOGICAL" } 200 OK @@ -639,6 +630,7 @@ X-Xss-Protection: 0 "datasetId": "bigquerydataset${uniqueId}", "projectId": "${projectId}" }, + "defaultCollation": "und:ci", "defaultEncryptionConfiguration": { "kmsKeyName": "projects/${projectId}/locations/us/keyRings/kmskeyring-${uniqueId}/cryptoKeys/kmscryptokey-${uniqueId}" }, @@ -648,6 +640,7 @@ X-Xss-Protection: 0 "etag": "abcdef0123A=", "friendlyName": "bigquerydataset-fullyconfigured-updated", "id": "000000000000000000000", + "isCaseInsensitive": false, "kind": "bigquery#dataset", "labels": { "cnrm-test": "true", @@ -702,6 +695,7 @@ X-Xss-Protection: 0 "datasetId": "bigquerydataset${uniqueId}", "projectId": "${projectId}" }, + "defaultCollation": "und:ci", "defaultEncryptionConfiguration": { "kmsKeyName": "projects/${projectId}/locations/us/keyRings/kmskeyring-${uniqueId}/cryptoKeys/kmscryptokey-${uniqueId}" }, @@ -711,6 +705,7 @@ X-Xss-Protection: 0 "etag": "abcdef0123A=", "friendlyName": "bigquerydataset-fullyconfigured-updated", "id": "000000000000000000000", + "isCaseInsensitive": false, "kind": "bigquery#dataset", "labels": { "cnrm-test": "true", @@ -764,6 +759,7 @@ X-Xss-Protection: 0 "datasetId": "bigquerydataset${uniqueId}", "projectId": "${projectId}" }, + "defaultCollation": "und:ci", "defaultEncryptionConfiguration": { "kmsKeyName": "projects/${projectId}/locations/us/keyRings/kmskeyring-${uniqueId}/cryptoKeys/kmscryptokey-${uniqueId}" }, @@ -773,6 +769,7 @@ X-Xss-Protection: 0 "etag": "abcdef0123A=", "friendlyName": "bigquerydataset-fullyconfigured-updated", "id": "000000000000000000000", + "isCaseInsensitive": false, "kind": "bigquery#dataset", "labels": { "cnrm-test": "true", @@ -788,7 +785,7 @@ X-Xss-Protection: 0 --- -DELETE https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&prettyPrint=false +DELETE https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&deleteContents=true&prettyPrint=false User-Agent: kcc/controller-manager 204 No Content diff --git a/tests/e2e/normalize.go b/tests/e2e/normalize.go index 8fc6fbcfd6..5e8ea890e0 100644 --- a/tests/e2e/normalize.go +++ b/tests/e2e/normalize.go @@ -705,6 +705,13 @@ func normalizeHTTPResponses(t *testing.T, events test.LogEntries) { // Compute resources visitor.sortSlices.Insert(".subnetworks") + // BigQuery resource + for _, event := range events { + if event.Request.Method == "PATCH" && strings.Contains(event.Request.URL, "bigquery.googleapis.com/bigquery/v2/") { + event.Request.ReplaceHeader("If-Match", "abcdef0123A=") + } + } + // Specific to Apigee visitor.replacePaths[".response.lastModifiedAt"] = "2024-04-01T12:34:56.123456Z" visitor.replacePaths[".response.createdAt"] = "2024-04-01T12:34:56.123456Z" diff --git a/tests/e2e/testdata/scenarios/fields/management/bigquery/dataset/set_unset/_http05.log b/tests/e2e/testdata/scenarios/fields/management/bigquery/dataset/set_unset/_http05.log index 30fd6eb04c..303e585230 100644 --- a/tests/e2e/testdata/scenarios/fields/management/bigquery/dataset/set_unset/_http05.log +++ b/tests/e2e/testdata/scenarios/fields/management/bigquery/dataset/set_unset/_http05.log @@ -1,6 +1,5 @@ -GET https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json -Content-Type: application/json -User-Agent: Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager +GET https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&prettyPrint=false +User-Agent: kcc/controller-manager 404 Not Found Cache-Control: private @@ -30,9 +29,9 @@ X-Xss-Protection: 0 --- -POST https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets?alt=json +POST https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets?alt=json&prettyPrint=false Content-Type: application/json -User-Agent: Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager +User-Agent: kcc/controller-manager { "access": [ @@ -48,69 +47,18 @@ User-Agent: Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 t "defaultEncryptionConfiguration": { "kmsKeyName": "projects/${projectId}/locations/us/keyRings/kmskeyring-${uniqueId}/cryptoKeys/kmscryptokey-${uniqueId}" }, - "defaultPartitionExpirationMs": 3600000, - "defaultTableExpirationMs": 3600000, - "description": "Fully Configured BigQuery Dataset", - "friendlyName": "bigquerydataset-fullyconfigured", - "isCaseInsensitive": true, - "labels": { - "managed-by-cnrm": "true" - }, - "location": "US", - "maxTimeTravelHours": "72" -} - -200 OK -Cache-Control: private -Content-Type: application/json; charset=UTF-8 -Server: ESF -Vary: Origin -Vary: X-Origin -Vary: Referer -X-Content-Type-Options: nosniff -X-Frame-Options: SAMEORIGIN -X-Xss-Protection: 0 - -{ - "access": [ - { - "role": "OWNER", - "specialGroup": "projectOwners" - } - ], - "creationTime": "123456789", - "datasetReference": { - "datasetId": "bigquerydataset${uniqueId}", - "projectId": "${projectId}" - }, - "defaultCollation": "und:ci", - "defaultEncryptionConfiguration": { - "kmsKeyName": "projects/${projectId}/locations/us/keyRings/kmskeyring-${uniqueId}/cryptoKeys/kmscryptokey-${uniqueId}" - }, "defaultPartitionExpirationMs": "3600000", "defaultTableExpirationMs": "3600000", "description": "Fully Configured BigQuery Dataset", - "etag": "abcdef0123A=", "friendlyName": "bigquerydataset-fullyconfigured", - "id": "000000000000000000000", "isCaseInsensitive": true, - "kind": "bigquery#dataset", "labels": { "managed-by-cnrm": "true" }, - "lastModifiedTime": "123456789", "location": "US", - "maxTimeTravelHours": "72", - "selfLink": "https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/bigquerydataset${uniqueId}", - "type": "DEFAULT" + "maxTimeTravelHours": "72" } ---- - -GET https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json -Content-Type: application/json -User-Agent: Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager - 200 OK Cache-Control: private Content-Type: application/json; charset=UTF-8 @@ -158,9 +106,8 @@ X-Xss-Protection: 0 --- -GET https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json -Content-Type: application/json -User-Agent: Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager +GET https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&prettyPrint=false +User-Agent: kcc/controller-manager 200 OK Cache-Control: private diff --git a/tests/e2e/testdata/scenarios/fields/management/bigquery/dataset/set_unset/_http06.log b/tests/e2e/testdata/scenarios/fields/management/bigquery/dataset/set_unset/_http06.log index 442cf64e06..eef39fb7ec 100644 --- a/tests/e2e/testdata/scenarios/fields/management/bigquery/dataset/set_unset/_http06.log +++ b/tests/e2e/testdata/scenarios/fields/management/bigquery/dataset/set_unset/_http06.log @@ -1,6 +1,5 @@ -GET https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json -Content-Type: application/json -User-Agent: Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager +GET https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&prettyPrint=false +User-Agent: kcc/controller-manager 200 OK Cache-Control: private @@ -49,46 +48,10 @@ X-Xss-Protection: 0 --- -PUT https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json +PATCH https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&prettyPrint=false Content-Type: application/json -User-Agent: Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager - -{ - "access": [ - { - "role": "OWNER", - "specialGroup": "projectOwners" - } - ], - "datasetReference": { - "datasetId": "bigquerydataset${uniqueId}" - }, - "defaultCollation": "und:ci", - "defaultEncryptionConfiguration": { - "kmsKeyName": "projects/${projectId}/locations/us/keyRings/kmskeyring-${uniqueId}/cryptoKeys/kmscryptokey-${uniqueId}" - }, - "defaultPartitionExpirationMs": 3600000, - "defaultTableExpirationMs": 3600000, - "description": "Fully Configured BigQuery Dataset unset", - "friendlyName": "bigquerydataset-fullyconfigured", - "isCaseInsensitive": true, - "labels": { - "managed-by-cnrm": "true" - }, - "location": "US", - "maxTimeTravelHours": "72" -} - -200 OK -Cache-Control: private -Content-Type: application/json; charset=UTF-8 -Server: ESF -Vary: Origin -Vary: X-Origin -Vary: Referer -X-Content-Type-Options: nosniff -X-Frame-Options: SAMEORIGIN -X-Xss-Protection: 0 +If-Match: abcdef0123A= +User-Agent: kcc/controller-manager { "access": [ @@ -97,11 +60,6 @@ X-Xss-Protection: 0 "specialGroup": "projectOwners" } ], - "creationTime": "123456789", - "datasetReference": { - "datasetId": "bigquerydataset${uniqueId}", - "projectId": "${projectId}" - }, "defaultCollation": "und:ci", "defaultEncryptionConfiguration": { "kmsKeyName": "projects/${projectId}/locations/us/keyRings/kmskeyring-${uniqueId}/cryptoKeys/kmscryptokey-${uniqueId}" @@ -109,27 +67,14 @@ X-Xss-Protection: 0 "defaultPartitionExpirationMs": "3600000", "defaultTableExpirationMs": "3600000", "description": "Fully Configured BigQuery Dataset unset", - "etag": "abcdef0123A=", "friendlyName": "bigquerydataset-fullyconfigured", - "id": "000000000000000000000", "isCaseInsensitive": true, - "kind": "bigquery#dataset", "labels": { "managed-by-cnrm": "true" }, - "lastModifiedTime": "123456789", - "location": "US", - "maxTimeTravelHours": "72", - "selfLink": "https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/bigquerydataset${uniqueId}", - "type": "DEFAULT" + "maxTimeTravelHours": "72" } ---- - -GET https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json -Content-Type: application/json -User-Agent: Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager - 200 OK Cache-Control: private Content-Type: application/json; charset=UTF-8 diff --git a/tests/e2e/testdata/scenarios/fields/management/bigquery/dataset/set_unset/_http07.log b/tests/e2e/testdata/scenarios/fields/management/bigquery/dataset/set_unset/_http07.log index 07e7e2ddc9..e2a2df3c8a 100644 --- a/tests/e2e/testdata/scenarios/fields/management/bigquery/dataset/set_unset/_http07.log +++ b/tests/e2e/testdata/scenarios/fields/management/bigquery/dataset/set_unset/_http07.log @@ -1,6 +1,5 @@ -GET https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json -Content-Type: application/json -User-Agent: Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager +GET https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&prettyPrint=false +User-Agent: kcc/controller-manager 200 OK Cache-Control: private @@ -49,46 +48,10 @@ X-Xss-Protection: 0 --- -PUT https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json +PATCH https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&prettyPrint=false Content-Type: application/json -User-Agent: Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager - -{ - "access": [ - { - "role": "OWNER", - "specialGroup": "projectOwners" - } - ], - "datasetReference": { - "datasetId": "bigquerydataset${uniqueId}" - }, - "defaultCollation": "und:ci", - "defaultEncryptionConfiguration": { - "kmsKeyName": "projects/${projectId}/locations/us/keyRings/kmskeyring-${uniqueId}/cryptoKeys/kmscryptokey-${uniqueId}" - }, - "defaultPartitionExpirationMs": 3600000, - "defaultTableExpirationMs": 3600000, - "description": "Fully Configured BigQuery Dataset set", - "friendlyName": "bigquerydataset", - "isCaseInsensitive": true, - "labels": { - "managed-by-cnrm": "true" - }, - "location": "US", - "maxTimeTravelHours": "72" -} - -200 OK -Cache-Control: private -Content-Type: application/json; charset=UTF-8 -Server: ESF -Vary: Origin -Vary: X-Origin -Vary: Referer -X-Content-Type-Options: nosniff -X-Frame-Options: SAMEORIGIN -X-Xss-Protection: 0 +If-Match: abcdef0123A= +User-Agent: kcc/controller-manager { "access": [ @@ -97,11 +60,6 @@ X-Xss-Protection: 0 "specialGroup": "projectOwners" } ], - "creationTime": "123456789", - "datasetReference": { - "datasetId": "bigquerydataset${uniqueId}", - "projectId": "${projectId}" - }, "defaultCollation": "und:ci", "defaultEncryptionConfiguration": { "kmsKeyName": "projects/${projectId}/locations/us/keyRings/kmskeyring-${uniqueId}/cryptoKeys/kmscryptokey-${uniqueId}" @@ -109,27 +67,14 @@ X-Xss-Protection: 0 "defaultPartitionExpirationMs": "3600000", "defaultTableExpirationMs": "3600000", "description": "Fully Configured BigQuery Dataset set", - "etag": "abcdef0123A=", "friendlyName": "bigquerydataset", - "id": "000000000000000000000", "isCaseInsensitive": true, - "kind": "bigquery#dataset", "labels": { "managed-by-cnrm": "true" }, - "lastModifiedTime": "123456789", - "location": "US", - "maxTimeTravelHours": "72", - "selfLink": "https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/bigquerydataset${uniqueId}", - "type": "DEFAULT" + "maxTimeTravelHours": "72" } ---- - -GET https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json -Content-Type: application/json -User-Agent: Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager - 200 OK Cache-Control: private Content-Type: application/json; charset=UTF-8 diff --git a/tests/e2e/testdata/scenarios/fields/management/bigquery/dataset/set_unset/_object05.yaml b/tests/e2e/testdata/scenarios/fields/management/bigquery/dataset/set_unset/_object05.yaml index 4fe4a90f2a..020e0f463e 100644 --- a/tests/e2e/testdata/scenarios/fields/management/bigquery/dataset/set_unset/_object05.yaml +++ b/tests/e2e/testdata/scenarios/fields/management/bigquery/dataset/set_unset/_object05.yaml @@ -2,13 +2,13 @@ apiVersion: bigquery.cnrm.cloud.google.com/v1beta1 kind: BigQueryDataset metadata: annotations: + alpha.cnrm.cloud.google.com/reconciler: direct cnrm.cloud.google.com/management-conflict-prevention-policy: none cnrm.cloud.google.com/project-id: ${projectId} - cnrm.cloud.google.com/state-into-spec: absent finalizers: - cnrm.cloud.google.com/finalizer - cnrm.cloud.google.com/deletion-defender - generation: 2 + generation: 1 name: bigquerydataset${uniqueId} namespace: ${projectId} spec: @@ -28,7 +28,6 @@ spec: maxTimeTravelHours: "72" projectRef: external: ${projectId} - resourceID: bigquerydataset${uniqueId} status: conditions: - lastTransitionTime: "1970-01-01T00:00:00Z" @@ -38,6 +37,9 @@ status: type: Ready creationTime: "1970-01-01T00:00:00Z" etag: abcdef123456 + externalRef: projects/${projectId}/datasets/bigquerydataset${uniqueId} lastModifiedTime: "1970-01-01T00:00:00Z" - observedGeneration: 2 + observedGeneration: 1 + observedState: + location: US selfLink: https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/bigquerydataset${uniqueId} diff --git a/tests/e2e/testdata/scenarios/fields/management/bigquery/dataset/set_unset/_object06.yaml b/tests/e2e/testdata/scenarios/fields/management/bigquery/dataset/set_unset/_object06.yaml index 41cd3e6bce..c6dad1de7c 100644 --- a/tests/e2e/testdata/scenarios/fields/management/bigquery/dataset/set_unset/_object06.yaml +++ b/tests/e2e/testdata/scenarios/fields/management/bigquery/dataset/set_unset/_object06.yaml @@ -2,13 +2,13 @@ apiVersion: bigquery.cnrm.cloud.google.com/v1beta1 kind: BigQueryDataset metadata: annotations: + alpha.cnrm.cloud.google.com/reconciler: direct cnrm.cloud.google.com/management-conflict-prevention-policy: none cnrm.cloud.google.com/project-id: ${projectId} - cnrm.cloud.google.com/state-into-spec: absent finalizers: - cnrm.cloud.google.com/finalizer - cnrm.cloud.google.com/deletion-defender - generation: 3 + generation: 2 name: bigquerydataset${uniqueId} namespace: ${projectId} spec: @@ -22,7 +22,6 @@ spec: location: US projectRef: external: ${projectId} - resourceID: bigquerydataset${uniqueId} status: conditions: - lastTransitionTime: "1970-01-01T00:00:00Z" @@ -32,6 +31,9 @@ status: type: Ready creationTime: "1970-01-01T00:00:00Z" etag: abcdef123456 + externalRef: projects/${projectId}/datasets/bigquerydataset${uniqueId} lastModifiedTime: "1970-01-01T00:00:00Z" - observedGeneration: 3 + observedGeneration: 2 + observedState: + location: US selfLink: https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/bigquerydataset${uniqueId} diff --git a/tests/e2e/testdata/scenarios/fields/management/bigquery/dataset/set_unset/_object07.yaml b/tests/e2e/testdata/scenarios/fields/management/bigquery/dataset/set_unset/_object07.yaml index 5b47b4afb7..84c86a4cd6 100644 --- a/tests/e2e/testdata/scenarios/fields/management/bigquery/dataset/set_unset/_object07.yaml +++ b/tests/e2e/testdata/scenarios/fields/management/bigquery/dataset/set_unset/_object07.yaml @@ -2,13 +2,13 @@ apiVersion: bigquery.cnrm.cloud.google.com/v1beta1 kind: BigQueryDataset metadata: annotations: + alpha.cnrm.cloud.google.com/reconciler: direct cnrm.cloud.google.com/management-conflict-prevention-policy: none cnrm.cloud.google.com/project-id: ${projectId} - cnrm.cloud.google.com/state-into-spec: absent finalizers: - cnrm.cloud.google.com/finalizer - cnrm.cloud.google.com/deletion-defender - generation: 4 + generation: 3 name: bigquerydataset${uniqueId} namespace: ${projectId} spec: @@ -28,7 +28,6 @@ spec: maxTimeTravelHours: "72" projectRef: external: ${projectId} - resourceID: bigquerydataset${uniqueId} status: conditions: - lastTransitionTime: "1970-01-01T00:00:00Z" @@ -38,6 +37,9 @@ status: type: Ready creationTime: "1970-01-01T00:00:00Z" etag: abcdef123456 + externalRef: projects/${projectId}/datasets/bigquerydataset${uniqueId} lastModifiedTime: "1970-01-01T00:00:00Z" - observedGeneration: 4 + observedGeneration: 3 + observedState: + location: US selfLink: https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/bigquerydataset${uniqueId} diff --git a/tests/e2e/testdata/scenarios/fields/management/bigquery/dataset/set_unset/script.yaml b/tests/e2e/testdata/scenarios/fields/management/bigquery/dataset/set_unset/script.yaml index daef2b0a09..a88830c2be 100644 --- a/tests/e2e/testdata/scenarios/fields/management/bigquery/dataset/set_unset/script.yaml +++ b/tests/e2e/testdata/scenarios/fields/management/bigquery/dataset/set_unset/script.yaml @@ -65,6 +65,8 @@ apiVersion: bigquery.cnrm.cloud.google.com/v1beta1 kind: BigQueryDataset metadata: name: bigquerydataset${uniqueId} + annotations: + alpha.cnrm.cloud.google.com/reconciler: "direct" spec: description: "Fully Configured BigQuery Dataset" friendlyName: bigquerydataset-fullyconfigured @@ -89,6 +91,8 @@ apiVersion: bigquery.cnrm.cloud.google.com/v1beta1 kind: BigQueryDataset metadata: name: bigquerydataset${uniqueId} + annotations: + alpha.cnrm.cloud.google.com/reconciler: "direct" spec: description: "Fully Configured BigQuery Dataset unset" # friendlyName: bigquerydataset-fullyconfigured @@ -112,6 +116,8 @@ apiVersion: bigquery.cnrm.cloud.google.com/v1beta1 kind: BigQueryDataset metadata: name: bigquerydataset${uniqueId} + annotations: + alpha.cnrm.cloud.google.com/reconciler: "direct" spec: description: "Fully Configured BigQuery Dataset set" friendlyName: bigquerydataset diff --git a/tests/e2e/unified_test.go b/tests/e2e/unified_test.go index 1048429f11..b8f475b4f8 100644 --- a/tests/e2e/unified_test.go +++ b/tests/e2e/unified_test.go @@ -672,6 +672,13 @@ func runScenario(ctx context.Context, t *testing.T, testPause bool, fixture reso } } }) + // Specific to BigQuery + addSetStringReplacement(".access[].userByEmail", "user@google.com") + for _, event := range events { + if event.Request.Method == "PATCH" && strings.Contains(event.Request.URL, "bigquery.googleapis.com/bigquery/v2/") { + event.Request.ReplaceHeader("If-Match", "abcdef0123A=") + } + } // Specific to BigTable addSetStringReplacement(".instances[].createTime", "2024-04-01T12:34:56.123456Z")