From c66f4796915cef2852c8e619d3e754a6c57c9f48 Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Fri, 28 Jun 2024 20:01:15 +0200 Subject: [PATCH 01/30] component-repo with bincode serde - wip --- Cargo.lock | 343 ++++++++-------- Cargo.toml | 8 + golem-component-service-base/Cargo.toml | 11 + golem-component-service-base/src/lib.rs | 1 + .../src/repo/component.rs | 372 ++++++++++++++++++ .../src/repo/mod.rs | 4 +- .../tests/db/migration/postgres/001__init.sql | 14 + .../tests/db/migration/sqlite/001__init.sql | 14 + golem-component-service/Cargo.toml | 2 +- golem-component-service/src/lib.rs | 1 - golem-component-service/src/repo/component.rs | 298 -------------- golem-service-base/Cargo.toml | 3 +- golem-service-base/src/model.rs | 112 ++++-- golem-worker-service-base/Cargo.toml | 2 +- .../src/repo/api_definition.rs | 14 + .../src/repo/api_deployment.rs | 14 + golem-worker-service-base/src/repo/mod.rs | 14 + .../src/service/api_definition.rs | 14 + .../src/service/api_definition_lookup.rs | 14 + .../src/service/api_definition_validator.rs | 14 + .../src/service/api_deployment.rs | 14 + golem-worker-service-base/src/service/mod.rs | 15 +- 22 files changed, 785 insertions(+), 513 deletions(-) create mode 100644 golem-component-service-base/src/repo/component.rs rename golem-component-service/src/repo.rs => golem-component-service-base/src/repo/mod.rs (100%) create mode 100644 golem-component-service-base/tests/db/migration/postgres/001__init.sql create mode 100644 golem-component-service-base/tests/db/migration/sqlite/001__init.sql delete mode 100644 golem-component-service/src/repo/component.rs diff --git a/Cargo.lock b/Cargo.lock index e35a01ca3d..f1458531f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,7 +8,7 @@ version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" dependencies = [ - "lazy_static 1.4.0", + "lazy_static 1.5.0", "regex", ] @@ -298,7 +298,7 @@ dependencies = [ "async-trait", "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", "tokio", ] @@ -444,7 +444,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -505,7 +505,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -522,7 +522,7 @@ checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -611,9 +611,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.7.3" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7d844e282b4b56750b2d4e893b2205581ded8709fddd2b6aa5418c150ca877" +checksum = "a8a47f2fb521b70c11ce7369a6c5fa4bd6af7e5d62ec06303875bafe7c6ba245" dependencies = [ "aws-lc-sys", "mirai-annotations", @@ -623,9 +623,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.18.0" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3a2c29203f6bf296d01141cc8bb9dbd5ecd4c27843f2ee0767bcd5985a927da" +checksum = "2927c7af777b460b7ccd95f8b67acd7b4c04ec8896bf0c8e80ba30523cffc057" dependencies = [ "bindgen", "cc", @@ -1121,11 +1121,11 @@ version = "0.69.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "cexpr", "clang-sys", "itertools 0.12.1", - "lazy_static 1.4.0", + "lazy_static 1.5.0", "lazycell", "log", "prettyplease", @@ -1134,7 +1134,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.67", + "syn 2.0.68", "which 4.4.2", ] @@ -1161,9 +1161,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" dependencies = [ "serde 1.0.203", ] @@ -1355,7 +1355,7 @@ dependencies = [ "cargo-component-core", "cargo-config2", "cargo_metadata", - "clap 4.5.7", + "clap 4.5.8", "futures", "heck 0.5.0", "indexmap 2.2.6", @@ -1394,7 +1394,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf60eee7916f926d079ac6dc1851bfd8e2869bbdfc3ff997013cf7b802565f86" dependencies = [ "anyhow", - "clap 4.5.7", + "clap 4.5.8", "futures", "indexmap 2.2.6", "libc", @@ -1476,9 +1476,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.99" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695" +checksum = "ac367972e516d45567c7eafc73d24e1c193dcf200a8d94e9db7b3d38b349572d" dependencies = [ "jobserver", "libc", @@ -1555,9 +1555,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.7" +version = "4.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5db83dced34638ad474f39f250d7fea9598bdd239eaced1bdf45d597da0f433f" +checksum = "84b3edb18336f4df585bc9aa31dd99c036dfa5dc5e9a2939a722a188f3a8970d" dependencies = [ "clap_builder", "clap_derive", @@ -1569,15 +1569,15 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb9b20c0dd58e4c2e991c8d203bbeb76c11304d1011659686b5b644bc29aa478" dependencies = [ - "clap 4.5.7", + "clap 4.5.8", "log", ] [[package]] name = "clap_builder" -version = "4.5.7" +version = "4.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7e204572485eb3fbf28f871612191521df159bc3e15a9f5064c66dba3a8c05f" +checksum = "c1c09dd5ada6c6c78075d6fd0da3f90d8080651e2d6cc8eb2f1aaa4034ced708" dependencies = [ "anstream", "anstyle", @@ -1587,14 +1587,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.5" +version = "4.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c780290ccf4fb26629baa7a1081e68ced113f1d3ec302fa5948f1c381ebf06c6" +checksum = "2bac35c6dafb060fd4d275d9a4ffae97917c13a6327903a8be2153cd964f7085" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -1659,7 +1659,7 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" dependencies = [ - "lazy_static 1.4.0", + "lazy_static 1.5.0", "windows-sys 0.48.0", ] @@ -1688,7 +1688,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b1b9d958c2b1368a663f05538fc1b5975adce1e19f435acceae987aceeeb369" dependencies = [ - "lazy_static 1.4.0", + "lazy_static 1.5.0", "nom 5.1.3", "rust-ini", "serde 1.0.203", @@ -1705,7 +1705,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" dependencies = [ "encode_unicode", - "lazy_static 1.4.0", + "lazy_static 1.5.0", "libc", "unicode-width", "windows-sys 0.52.0", @@ -2024,7 +2024,7 @@ dependencies = [ "criterion-plot", "csv", "itertools 0.10.5", - "lazy_static 1.4.0", + "lazy_static 1.5.0", "num-traits 0.2.19", "oorandom", "plotters", @@ -2189,7 +2189,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f" dependencies = [ "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -2246,7 +2246,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -2268,7 +2268,7 @@ checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" dependencies = [ "darling_core 0.20.9", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -2349,7 +2349,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -2362,7 +2362,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -2513,7 +2513,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439a1c2ba5611ad3ed731280541d36d2e9c4ac5e7fb818a27b604bdc5a6aa65b" dependencies = [ - "lazy_static 1.4.0", + "lazy_static 1.5.0", "libc", "winapi", "wio", @@ -2553,9 +2553,9 @@ dependencies = [ [[package]] name = "either" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" dependencies = [ "serde 1.0.203", ] @@ -2639,7 +2639,7 @@ checksum = "de0d48a183585823424a4ce1aa132d174a6a81bd540895822eb4c8373a8e49e8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -2899,7 +2899,7 @@ dependencies = [ "futures-core", "futures-sink", "nanorand", - "spin 0.9.8", + "spin", ] [[package]] @@ -2920,7 +2920,7 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2845a73bbd781e691ab7c2a028c579727cd254942e8ced57ff73e0eafd60de87" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "byteorder", "core-foundation", "core-graphics", @@ -2929,7 +2929,7 @@ dependencies = [ "dwrote", "float-ord", "freetype-sys", - "lazy_static 1.4.0", + "lazy_static 1.5.0", "libc", "log", "pathfinder_geometry", @@ -2966,7 +2966,7 @@ checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -3148,7 +3148,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -3217,7 +3217,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27d12c0aed7f1e24276a241aadc4cb8ea9f83000f34bc062b7cc2d51e3b0fabd" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "debugid", "fxhash", "serde 1.0.203", @@ -3344,7 +3344,7 @@ dependencies = [ "async-recursion", "async-trait", "chrono", - "clap 4.5.7", + "clap 4.5.8", "clap-verbosity-flag", "cli-table", "colored", @@ -3436,7 +3436,7 @@ dependencies = [ "http 0.2.12", "humantime-serde", "iso8601-timestamp", - "lazy_static 1.4.0", + "lazy_static 1.5.0", "poem", "poem-openapi", "prometheus", @@ -3468,7 +3468,7 @@ dependencies = [ "golem-worker-executor-base", "http 0.2.12", "http 1.1.0", - "lazy_static 1.4.0", + "lazy_static 1.5.0", "prometheus", "serde 1.0.203", "thiserror", @@ -3498,7 +3498,7 @@ dependencies = [ "golem-wasm-rpc", "http 0.2.12", "humantime-serde", - "lazy_static 1.4.0", + "lazy_static 1.5.0", "mappable-rc", "opentelemetry 0.23.0", "opentelemetry-prometheus 0.16.0", @@ -3527,15 +3527,19 @@ name = "golem-component-service-base" version = "0.0.0" dependencies = [ "async-trait", + "bincode", + "bytes 1.6.0", "golem-api-grpc", "golem-common", "golem-service-base", "golem-wasm-ast", "http 0.2.12", "serde 1.0.203", + "sqlx", "thiserror", "tonic", "tracing", + "uuid", ] [[package]] @@ -3565,7 +3569,7 @@ version = "0.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93808434ae66e60e3316b99a3c2333a112bdb0662efb9a52d81ba7713e1d5808" dependencies = [ - "clap 4.5.7", + "clap 4.5.8", "convert_case 0.6.0", "fmt", "indexmap 2.2.6", @@ -3600,6 +3604,7 @@ dependencies = [ "aws-config", "aws-sdk-s3", "bigdecimal", + "bincode", "futures", "golem-api-grpc", "golem-common", @@ -3667,7 +3672,7 @@ dependencies = [ "async-dropper-simple", "async-scoped", "async-trait", - "clap 4.5.7", + "clap 4.5.8", "cli-table", "colored", "console-subscriber", @@ -3743,7 +3748,7 @@ dependencies = [ "cargo-component", "cargo-component-core", "cargo_toml", - "clap 4.5.7", + "clap 4.5.8", "dir-diff", "fs_extra", "golem-wasm-ast", @@ -3757,7 +3762,7 @@ dependencies = [ "quote", "regex", "serde 1.0.203", - "syn 2.0.67", + "syn 2.0.68", "tempdir", "tokio", "toml 0.8.14", @@ -3821,7 +3826,7 @@ dependencies = [ "aws-config", "aws-sdk-s3", "bincode", - "bitflags 2.5.0", + "bitflags 2.6.0", "bytes 1.6.0", "cap-fs-ext", "cap-std", @@ -3854,7 +3859,7 @@ dependencies = [ "hyper 1.3.1", "io-extras", "iso8601-timestamp", - "lazy_static 1.4.0", + "lazy_static 1.5.0", "log", "md5", "metrohash", @@ -3915,7 +3920,7 @@ dependencies = [ "http 1.1.0", "humantime-serde", "hyper 1.3.1", - "lazy_static 1.4.0", + "lazy_static 1.5.0", "nom 7.1.3", "openapiv3", "opentelemetry 0.23.0", @@ -3967,7 +3972,7 @@ dependencies = [ "http 1.1.0", "humantime-serde", "hyper 1.3.1", - "lazy_static 1.4.0", + "lazy_static 1.5.0", "nom 7.1.3", "openapiv3", "opentelemetry 0.23.0", @@ -4628,7 +4633,7 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fddf93031af70e75410a2511ec04d49e758ed2f26dad3404a934e0fb45cc12a" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "crossterm", "dyn-clone", "fuzzy-matcher", @@ -4655,7 +4660,7 @@ dependencies = [ "anyhow", "assert2", "async-trait", - "clap 4.5.7", + "clap 4.5.8", "console-subscriber", "ctor", "golem-api-grpc", @@ -4845,7 +4850,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d8fe85bd70ff715f31ce8c739194b423d79811a19602115d611a3ec85d6200" dependencies = [ - "lazy_static 1.4.0", + "lazy_static 1.5.0", "once_cell", "pest", "pest_derive", @@ -4885,7 +4890,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "363387f0019d714aa60cc30ab4fe501a747f4c08fc58f069dd14be971bd495a0" dependencies = [ "byteorder", - "lazy_static 1.4.0", + "lazy_static 1.5.0", "linux-keyutils", "secret-service", "security-framework", @@ -4983,7 +4988,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -5022,11 +5027,11 @@ checksum = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin 0.5.2", + "spin", ] [[package]] @@ -5062,9 +5067,9 @@ checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "libloading" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" +checksum = "e310b3a6b5907f99202fcdb4960ff45b93735d7c7d96b760fcff8db2dc0e103d" dependencies = [ "cfg-if", "windows-targets 0.52.5", @@ -5082,7 +5087,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "libc", ] @@ -5102,7 +5107,7 @@ name = "libtest-mimic" version = "0.7.0" source = "git+https://github.com/senia-psm/libtest-mimic.git?branch=async_and_context_tests#904a9367691b7eb47dc25a876391ee4a958c0319" dependencies = [ - "clap 4.5.7", + "clap 4.5.8", "escape8259", "termcolor", "threadpool", @@ -5121,7 +5126,7 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "761e49ec5fd8a5a463f9b84e877c373d888935b71c6be78f3767fe2ae6bed18e" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "libc", ] @@ -5149,9 +5154,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.21" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "logos" @@ -5170,11 +5175,11 @@ checksum = "8e31badd9de5131fdf4921f6473d457e3dd85b11b7f091ceb50e4df7c3eeb12a" dependencies = [ "beef", "fnv", - "lazy_static 1.4.0", + "lazy_static 1.5.0", "proc-macro2", "quote", "regex-syntax 0.8.4", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -5306,7 +5311,7 @@ checksum = "dcf09caffaac8068c346b6df2a7fc27a177fd20b39421a39ce0a211bde679a6c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -5373,7 +5378,7 @@ dependencies = [ "log", "memchr", "mime", - "spin 0.9.8", + "spin", "version_check", ] @@ -5390,7 +5395,7 @@ dependencies = [ "httparse", "memchr", "mime", - "spin 0.9.8", + "spin", "tokio", "version_check", ] @@ -5454,7 +5459,7 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "cfg-if", "libc", ] @@ -5465,7 +5470,7 @@ version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "cfg-if", "cfg_aliases", "libc", @@ -5494,9 +5499,9 @@ dependencies = [ [[package]] name = "nonempty-collections" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce843257ad9150ef688ce56753ef461fb2de929fa980fd5692828d1df5fe6f4d" +checksum = "2220c0aadad0ee210950152e56eb14a1a4415367151d1f2a2cad24ab5eaecb58" [[package]] name = "normpath" @@ -5542,9 +5547,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c165a9ab64cf766f73521c0dd2cfdff64f488b8f0b3e621face3462d3db536d7" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits 0.2.19", @@ -5557,7 +5562,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" dependencies = [ "byteorder", - "lazy_static 1.4.0", + "lazy_static 1.5.0", "libm", "num-integer", "num-iter", @@ -5698,7 +5703,7 @@ version = "0.10.64" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "cfg-if", "foreign-types 0.3.2", "libc", @@ -5715,7 +5720,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -5837,7 +5842,7 @@ dependencies = [ "futures-executor", "futures-util", "glob", - "lazy_static 1.4.0", + "lazy_static 1.5.0", "once_cell", "opentelemetry 0.23.0", "ordered-float 4.2.0", @@ -6053,7 +6058,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -6112,7 +6117,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -6191,7 +6196,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -6263,7 +6268,7 @@ dependencies = [ "chrono", "font-kit", "image", - "lazy_static 1.4.0", + "lazy_static 1.5.0", "num-traits 0.2.19", "pathfinder_geometry", "plotters-backend", @@ -6371,7 +6376,7 @@ dependencies = [ "proc-macro-crate 3.1.0", "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -6418,7 +6423,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.67", + "syn 2.0.68", "thiserror", ] @@ -6548,7 +6553,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -6596,7 +6601,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", "version_check", "yansi 1.0.1", ] @@ -6607,9 +6612,9 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "731e0d9356b0c25f16f33b5be79b1c57b562f141ebfcdb0ad8ac2c13a24293b4" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "hex", - "lazy_static 1.4.0", + "lazy_static 1.5.0", "procfs-core", "rustix 0.38.34", ] @@ -6620,7 +6625,7 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d3554923a69f4ce04c4a754260c338f505ce22642d3830e049a399fc2059a29" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "hex", ] @@ -6632,7 +6637,7 @@ checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1" dependencies = [ "cfg-if", "fnv", - "lazy_static 1.4.0", + "lazy_static 1.5.0", "libc", "memchr", "parking_lot", @@ -6643,14 +6648,14 @@ dependencies = [ [[package]] name = "proptest" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31b476131c3c86cb68032fdc5cb6d5a1045e3e42d96b69fa599fd77701e1f5bf" +checksum = "b4c2511913b88df1637da85cc8d96ec8e43a3f8bb8ccb71ee1ac240d6f3df58d" dependencies = [ "bit-set", "bit-vec", - "bitflags 2.5.0", - "lazy_static 1.4.0", + "bitflags 2.6.0", + "lazy_static 1.5.0", "num-traits 0.2.19", "rand 0.8.5", "rand_chacha", @@ -6688,7 +6693,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.67", + "syn 2.0.68", "tempfile", ] @@ -6702,7 +6707,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -6971,7 +6976,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", ] [[package]] @@ -7151,7 +7156,7 @@ dependencies = [ "cfg-if", "getrandom", "libc", - "spin 0.9.8", + "spin", "untrusted", "windows-sys 0.52.0", ] @@ -7253,7 +7258,7 @@ version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "errno", "itoa", "libc", @@ -7439,7 +7444,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -7527,7 +7532,7 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "core-foundation", "core-foundation-sys", "libc", @@ -7574,7 +7579,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a3a4e0ea8a88553209f6cc6cfe8724ecad22e1acf372793c27d995290fe74f8" dependencies = [ - "lazy_static 1.4.0", + "lazy_static 1.5.0", "num-traits 0.1.43", "regex", "serde 0.8.23", @@ -7608,7 +7613,7 @@ checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -7619,14 +7624,14 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] name = "serde_json" -version = "1.0.117" +version = "1.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" +checksum = "d947f6b3163d8857ea16c4fa0dd4840d52f3041039a85decd46867eb1abef2e4" dependencies = [ "itoa", "ryu", @@ -7683,7 +7688,7 @@ checksum = "75dde5a1d2ed78dfc411fc45592f72d3694436524d3353683ecb3d22009731dc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -7694,7 +7699,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -7767,7 +7772,7 @@ dependencies = [ "darling 0.20.9", "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -7830,7 +7835,7 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" dependencies = [ - "lazy_static 1.4.0", + "lazy_static 1.5.0", ] [[package]] @@ -7991,12 +7996,6 @@ dependencies = [ "smallvec", ] -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - [[package]] name = "spin" version = "0.9.8" @@ -8143,7 +8142,7 @@ checksum = "1ed31390216d20e538e447a7a9b959e06ed9fc51c37b514b46eb758016ecd418" dependencies = [ "atoi", "base64 0.21.7", - "bitflags 2.5.0", + "bitflags 2.6.0", "byteorder", "bytes 1.6.0", "chrono", @@ -8187,7 +8186,7 @@ checksum = "7c824eb80b894f926f89a0b9da0c7f435d27cdd35b8c655b114e58223918577e" dependencies = [ "atoi", "base64 0.21.7", - "bitflags 2.5.0", + "bitflags 2.6.0", "byteorder", "chrono", "crc", @@ -8302,9 +8301,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "strum" -version = "0.26.2" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" [[package]] name = "strum_macros" @@ -8316,14 +8315,14 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] name = "subtle" -version = "2.6.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d0208408ba0c3df17ed26eb06992cb1a1268d41b2c0e12e65203fbe3972cee5" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" @@ -8338,9 +8337,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.67" +version = "2.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff8655ed1d86f3af4ee3fd3263786bc14245ad17c4c7e85ba7187fb3ae028c90" +checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" dependencies = [ "proc-macro2", "quote", @@ -8404,7 +8403,7 @@ version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b858526d22750088a9b3cf2e3c2aacebd5377f13adeec02860c30d09113010a6" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "cap-fs-ext", "cap-std", "fd-lock", @@ -8522,7 +8521,7 @@ checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -8596,9 +8595,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +checksum = "c55115c6fbe2d2bef26eb09ad74bde02d8255476fc0c7b515ef09fbb35742d82" dependencies = [ "tinyvec_macros", ] @@ -8647,7 +8646,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -8885,7 +8884,7 @@ dependencies = [ "proc-macro2", "prost-build", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -8941,7 +8940,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" dependencies = [ "base64 0.21.7", - "bitflags 2.5.0", + "bitflags 2.6.0", "bytes 1.6.0", "http 1.1.0", "http-body 1.0.0", @@ -8985,7 +8984,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -9068,7 +9067,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04659ddb06c87d233c566112c1c9c5b9e98256d9af50ec3bc9c8327f873a7568" dependencies = [ "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -9280,9 +9279,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.8.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" +checksum = "5de17fd2f7da591098415cff336e12965a28061ddace43b59cb3c430179c9439" dependencies = [ "getrandom", "serde 1.0.203", @@ -9403,7 +9402,7 @@ dependencies = [ "async-recursion", "async-trait", "bytes 1.6.0", - "clap 4.5.7", + "clap 4.5.8", "dialoguer", "dirs 5.0.1", "futures-util", @@ -9578,7 +9577,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", "wasm-bindgen-shared", ] @@ -9612,7 +9611,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -9705,9 +9704,9 @@ dependencies = [ [[package]] name = "wasm-encoder" -version = "0.211.1" +version = "0.212.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e7d931a1120ef357f32b74547646b6fa68ea25e377772b72874b131a9ed70d4" +checksum = "501940df4418b8929eb6d52f1aade1fdd15a5b86c92453cb696e3c906bd3fc33" dependencies = [ "leb128", ] @@ -9792,7 +9791,7 @@ version = "0.121.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9dbe55c8f9d0dbd25d9447a5a889ff90c0cc3feaa7395310d3d826b2c703eaab" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "indexmap 2.2.6", "semver", ] @@ -9804,7 +9803,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e19bb9f8ab07616da582ef8adb24c54f1424c7ec876720b7da9db8ec0626c92c" dependencies = [ "ahash", - "bitflags 2.5.0", + "bitflags 2.6.0", "hashbrown 0.14.5", "indexmap 2.2.6", "semver", @@ -9817,7 +9816,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd921789c9dcc495f589cb37d200155dee65b4a4beeb853323b5e24e0a5f9c58" dependencies = [ "ahash", - "bitflags 2.5.0", + "bitflags 2.6.0", "hashbrown 0.14.5", "indexmap 2.2.6", "semver", @@ -9831,7 +9830,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07035cc9a9b41e62d3bb3a3815a66ab87c993c06fe1cf6b2a3f2a18499d937db" dependencies = [ "ahash", - "bitflags 2.5.0", + "bitflags 2.6.0", "hashbrown 0.14.5", "indexmap 2.2.6", "semver", @@ -9947,7 +9946,7 @@ dependencies = [ "anyhow", "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", "wasmtime-component-util", "wasmtime-wit-bindgen", "wit-parser 0.207.0", @@ -10065,7 +10064,7 @@ source = "git+https://github.com/golemcloud/wasmtime.git?branch=golem-wasmtime-v dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -10075,7 +10074,7 @@ source = "git+https://github.com/golemcloud/wasmtime.git?branch=golem-wasmtime-v dependencies = [ "anyhow", "async-trait", - "bitflags 2.5.0", + "bitflags 2.6.0", "bytes 1.6.0", "cap-fs-ext", "cap-net-ext", @@ -10158,24 +10157,24 @@ dependencies = [ [[package]] name = "wast" -version = "211.0.1" +version = "212.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b25506dd82d00da6b14a87436b3d52b1d264083fa79cdb72a0d1b04a8595ccaa" +checksum = "4606a05fb0aae5d11dd7d8280a640d88a63ee019360ba9be552da3d294b8d1f5" dependencies = [ "bumpalo", "leb128", "memchr", "unicode-width", - "wasm-encoder 0.211.1", + "wasm-encoder 0.212.0", ] [[package]] name = "wat" -version = "1.211.1" +version = "1.212.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb716ca6c86eecac2d82541ffc39860118fc0af9309c4f2670637bea2e1bdd7d" +checksum = "c74ca7f93f11a5d6eed8499f2a8daaad6e225cab0151bc25a091fff3b987532f" dependencies = [ - "wast 211.0.1", + "wast 212.0.0", ] [[package]] @@ -10245,7 +10244,7 @@ source = "git+https://github.com/golemcloud/wasmtime.git?branch=golem-wasmtime-v dependencies = [ "anyhow", "async-trait", - "bitflags 2.5.0", + "bitflags 2.6.0", "thiserror", "tracing", "wasmtime", @@ -10262,7 +10261,7 @@ dependencies = [ "proc-macro2", "quote", "shellexpand", - "syn 2.0.67", + "syn 2.0.68", "witx", ] @@ -10273,7 +10272,7 @@ source = "git+https://github.com/golemcloud/wasmtime.git?branch=golem-wasmtime-v dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", "wiggle-generate", ] @@ -10528,7 +10527,7 @@ version = "0.36.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9643b83820c0cd246ecabe5fa454dd04ba4fa67996369466d0747472d337346" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "windows-sys 0.52.0", ] @@ -10569,7 +10568,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29c7526379ace8709ee9ab9f2bb50f112d95581063a59ef3097d9c10153886c9" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", ] [[package]] @@ -10607,7 +10606,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fef7dd0e47f5135dd8739ccc5b188ab8b7e27e1d64df668aa36680f0b8646db8" dependencies = [ "anyhow", - "bitflags 2.5.0", + "bitflags 2.6.0", "indexmap 2.2.6", "log", "serde 1.0.203", @@ -10626,7 +10625,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25a2bb5b039f9cb03425e1d5a6e54b441ca4ca1b1d4fa6a0924db67a55168f99" dependencies = [ "anyhow", - "bitflags 2.5.0", + "bitflags 2.6.0", "indexmap 2.2.6", "log", "serde 1.0.203", @@ -10836,7 +10835,7 @@ checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -10856,7 +10855,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 2f9e5df40d..8d4d090506 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -135,6 +135,14 @@ rand = "0.8.5" serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0" } serde_yaml = { version = "0.9.33 " } +sqlx = { version = "0.7", features = [ + "runtime-tokio", + "sqlite", + "postgres", + "uuid", + "migrate", + "chrono", +] } strum = "0.26.1" strum_macros = "0.26.1" tap = "1.0.1" diff --git a/golem-component-service-base/Cargo.toml b/golem-component-service-base/Cargo.toml index 5a7511ffaa..dfa783b758 100644 --- a/golem-component-service-base/Cargo.toml +++ b/golem-component-service-base/Cargo.toml @@ -12,8 +12,19 @@ golem-service-base = { path = "../golem-service-base" } golem-wasm-ast = { workspace = true } async-trait = { workspace = true } +bincode = { workspace = true } +bytes = { workspace = true } http_02 = { workspace = true } serde = { workspace = true } +sqlx = { workspace = true, features = [ + "runtime-tokio", + "sqlite", + "postgres", + "uuid", + "migrate", + "chrono", +] } thiserror = { workspace = true } tonic = { workspace = true } tracing = { workspace = true } +uuid = { workspace = true } \ No newline at end of file diff --git a/golem-component-service-base/src/lib.rs b/golem-component-service-base/src/lib.rs index 35d914db16..e369b480ee 100644 --- a/golem-component-service-base/src/lib.rs +++ b/golem-component-service-base/src/lib.rs @@ -13,4 +13,5 @@ // limitations under the License. pub mod config; +pub mod repo; pub mod service; diff --git a/golem-component-service-base/src/repo/component.rs b/golem-component-service-base/src/repo/component.rs new file mode 100644 index 0000000000..08196f1795 --- /dev/null +++ b/golem-component-service-base/src/repo/component.rs @@ -0,0 +1,372 @@ +// Copyright 2024 Golem Cloud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::fmt::Display; +use std::ops::Deref; +use std::result::Result; +use std::sync::Arc; + +use async_trait::async_trait; +use golem_common::model::ComponentId; +use sqlx::{Database, Pool}; +use uuid::Uuid; + +use crate::repo::RepoError; +use golem_service_base::model::*; + +#[derive(sqlx::FromRow, Debug, Clone)] +pub struct ComponentRecord { + pub namespace: String, + pub component_id: Uuid, + pub name: String, + pub size: i32, + pub version: i64, + pub user_component: String, + pub protected_component: String, + pub protector_version: Option, + pub metadata: Vec, +} + +impl TryFrom for Component { + type Error = String; + fn try_from(value: ComponentRecord) -> Result { + let metadata: ComponentMetadata = record_metadata_serde::deserialize(&value.metadata)?; + let versioned_component_id: VersionedComponentId = VersionedComponentId { + component_id: ComponentId(value.component_id), + version: value.version as u64, + }; + let protected_component_id: ProtectedComponentId = ProtectedComponentId { + versioned_component_id: versioned_component_id.clone(), + }; + let user_component_id: UserComponentId = UserComponentId { + versioned_component_id: versioned_component_id.clone(), + }; + Ok(Component { + component_name: ComponentName(value.name), + component_size: value.size as u64, + metadata, + versioned_component_id, + user_component_id, + protected_component_id, + }) + } +} + +impl ComponentRecord { + pub fn new( + namespace: Namespace, + component: Component, + ) -> Result { + let metadata = record_metadata_serde::serialize(&component.metadata)?; + Ok(Self { + namespace: namespace.to_string(), + component_id: component.versioned_component_id.component_id.0, + name: component.component_name.0, + size: component.component_size as i32, + version: component.versioned_component_id.version as i64, + user_component: component.versioned_component_id.slug(), + protected_component: component.protected_component_id.slug(), + protector_version: None, + metadata: metadata.into(), + }) + } +} + +#[async_trait] +pub trait ComponentRepo { + async fn upsert(&self, component: &ComponentRecord) -> Result<(), RepoError>; + + async fn get( + &self, + namespace: &str, + component_id: &Uuid, + ) -> Result, RepoError>; + + async fn get_all(&self) -> Result, RepoError>; + + async fn get_latest_version( + &self, + namespace: &str, + component_id: &Uuid, + ) -> Result, RepoError>; + + async fn get_by_version( + &self, + namespace: &str, + component_id: &Uuid, + version: u64, + ) -> Result, RepoError>; + + async fn get_by_name( + &self, + namespace: &str, + name: &str, + ) -> Result, RepoError>; + + async fn delete(&self, namespace: &str, component_id: &Uuid) -> Result<(), RepoError>; +} + +pub struct DbComponentRepo { + db_pool: Arc>, +} + +impl DbComponentRepo { + pub fn new(db_pool: Arc>) -> Self { + Self { db_pool } + } +} + +#[async_trait] +impl ComponentRepo for DbComponentRepo { + async fn upsert(&self, component: &ComponentRecord) -> Result<(), RepoError> { + sqlx::query( + r#" + INSERT INTO components + (namespace, component_id, version, name, size, user_component, protected_component, protector_version, metadata) + VALUES + ($1, $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT (namespace, component_id, version) DO UPDATE + SET name = $4, + size = $5, + user_component = $6, + protected_component = $7, + protector_version = $8, + metadata = $9 + "#, + ) + .bind(component.namespace.clone()) + .bind(component.component_id) + .bind(component.version) + .bind(component.name.clone()) + .bind(component.size) + .bind(component.user_component.clone()) + .bind(component.protected_component.clone()) + .bind(component.protector_version) + .bind(component.metadata.clone()) + .execute(self.db_pool.deref()) + .await?; + + Ok(()) + } + + async fn get( + &self, + namespace: &str, + component_id: &Uuid, + ) -> Result, RepoError> { + sqlx::query_as::<_, ComponentRecord>("SELECT namespace, component_id, version, name, size, user_component, protected_component, protector_version, metadata FROM components WHERE namespace = $1 AND component_id = $2") + .bind(namespace) + .bind(component_id) + .fetch_all(self.db_pool.deref()) + .await + .map_err(|e| e.into()) + } + + async fn get_all(&self) -> Result, RepoError> { + sqlx::query_as::<_, ComponentRecord>("SELECT namespace, component_id, version, name, size, user_component, protected_component, protector_version, metadata FROM components") + .fetch_all(self.db_pool.deref()) + .await + .map_err(|e| e.into()) + } + + async fn get_latest_version( + &self, + namespace: &str, + component_id: &Uuid, + ) -> Result, RepoError> { + sqlx::query_as::<_, ComponentRecord>( + "SELECT namespace, component_id, version, name, size, user_component, protected_component, protector_version, metadata FROM components WHERE namespace = $1 AND component_id = $2 ORDER BY version DESC LIMIT 1", + ).bind(namespace) + .bind(component_id) + .fetch_optional(self.db_pool.deref()) + .await + .map_err(|e| e.into()) + } + + async fn get_by_version( + &self, + namespace: &str, + component_id: &Uuid, + version: u64, + ) -> Result, RepoError> { + sqlx::query_as::<_, ComponentRecord>( + "SELECT namespace, component_id, version, name, size, user_component, protected_component, protector_version, metadata FROM components WHERE namespace = $1 AND component_id = $2 AND version = $3", + ) + .bind(namespace) + .bind(component_id) + .bind(version as i64) + .fetch_optional(self.db_pool.deref()) + .await + .map_err(|e| e.into()) + } + + async fn get_by_name( + &self, + namespace: &str, + name: &str, + ) -> Result, RepoError> { + sqlx::query_as::<_, ComponentRecord>( + "SELECT namespace, component_id, version, name, size, user_component, protected_component, protector_version, metadata FROM components WHERE namespace = $1 AND name = $2", + ) + .bind(namespace) + .bind(name) + .fetch_all(self.db_pool.deref()) + .await + .map_err(|e| e.into()) + } + + async fn delete(&self, namespace: &str, component_id: &Uuid) -> Result<(), RepoError> { + sqlx::query("DELETE FROM components WHERE namespace = $1 AND component_id = $2") + .bind(namespace) + .bind(component_id) + .execute(self.db_pool.deref()) + .await?; + Ok(()) + } +} + +#[async_trait] +impl ComponentRepo for DbComponentRepo { + async fn upsert(&self, component: &ComponentRecord) -> Result<(), RepoError> { + sqlx::query( + r#" + INSERT INTO components + (namespace, component_id, version, name, size, user_component, protected_component, protector_version, metadata) + VALUES + ($1, $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT (namespace, component_id, version) DO UPDATE + SET name = $4, + size = $5, + user_component = $6, + protected_component = $7, + protector_version = $8, + metadata = $9 + "#, + ) + .bind(component.namespace.clone()) + .bind(component.component_id) + .bind(component.version) + .bind(component.name.clone()) + .bind(component.size) + .bind(component.user_component.clone()) + .bind(component.protected_component.clone()) + .bind(component.protector_version) + .bind(component.metadata.clone()) + .execute(self.db_pool.deref()) + .await?; + + Ok(()) + } + + async fn get_all(&self) -> Result, RepoError> { + sqlx::query_as::<_, ComponentRecord>("SELECT namespace, component_id, name, size, version, user_component, protected_component, protector_version, metadata FROM components") + .fetch_all(self.db_pool.deref()) + .await + .map_err(|e| e.into()) + } + + async fn get( + &self, + namespace: &str, + component_id: &Uuid, + ) -> Result, RepoError> { + sqlx::query_as::<_, ComponentRecord>("SELECT namespace, component_id, name, size, version, user_component, protected_component, protector_version, metadata FROM components WHERE namespace = $1 AND component_id = $2") + .bind(namespace) + .bind(component_id) + .fetch_all(self.db_pool.deref()) + .await + .map_err(|e| e.into()) + } + + async fn get_by_name( + &self, + namespace: &str, + name: &str, + ) -> Result, RepoError> { + sqlx::query_as::<_, ComponentRecord>( + "SELECT namespace, component_id, name, size, version, user_component, protected_component, protector_version, metadata FROM components WHERE namespace = $1 AND name = $2", + ) + .bind(namespace) + .bind(name) + .fetch_all(self.db_pool.deref()) + .await + .map_err(|e| e.into()) + } + + async fn get_latest_version( + &self, + namespace: &str, + component_id: &Uuid, + ) -> Result, RepoError> { + sqlx::query_as::<_, ComponentRecord>( + "SELECT namespace, component_id, name, size, version, user_component, protected_component, protector_version, metadata FROM components WHERE namespace = $1 AND component_id = $2 ORDER BY version DESC LIMIT 1", + ) + .bind(namespace) + .bind(component_id) + .fetch_optional(self.db_pool.deref()) + .await + .map_err(|e| e.into()) + } + + async fn get_by_version( + &self, + namespace: &str, + component_id: &Uuid, + version: u64, + ) -> Result, RepoError> { + sqlx::query_as::<_, ComponentRecord>( + "SELECT namespace, component_id, name, size, version, user_component, protected_component, protector_version, metadata FROM components WHERE namespace = $1 AND component_id = $2 AND version = $3", + ) + .bind(namespace) + .bind(component_id) + .bind(version as i64) + .fetch_optional(self.db_pool.deref()) + .await + .map_err(|e| e.into()) + } + + async fn delete(&self, namespace: &str, component_id: &Uuid) -> Result<(), RepoError> { + sqlx::query("DELETE FROM components WHERE namespace = $1 AND component_id = $2") + .bind(namespace) + .bind(component_id) + .execute(self.db_pool.deref()) + .await?; + Ok(()) + } +} + +pub mod record_metadata_serde { + use bincode::{Decode, Encode}; + use bytes::Bytes; + use golem_common::serialization::serialize_with_version; + pub const SERIALIZATION_VERSION_V1: u8 = 1u8; + + pub fn serialize(routes: &T) -> Result { + serialize_with_version(routes, SERIALIZATION_VERSION_V1) + } + + pub fn deserialize(bytes: &[u8]) -> Result { + let (version, data) = bytes.split_at(1); + + match version[0] { + SERIALIZATION_VERSION_V1 => { + let (routes, _) = bincode::decode_from_slice(data, bincode::config::standard()) + .map_err(|e| format!("Failed to deserialize value: {e}"))?; + + Ok(routes) + } + _ => Err("Unsupported serialization version".to_string()), + } + } +} diff --git a/golem-component-service/src/repo.rs b/golem-component-service-base/src/repo/mod.rs similarity index 100% rename from golem-component-service/src/repo.rs rename to golem-component-service-base/src/repo/mod.rs index 96b5627c62..71ba9a9a10 100644 --- a/golem-component-service/src/repo.rs +++ b/golem-component-service-base/src/repo/mod.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::fmt::Display; - pub mod component; +use std::fmt::Display; + #[derive(Debug)] pub enum RepoError { Internal(String), diff --git a/golem-component-service-base/tests/db/migration/postgres/001__init.sql b/golem-component-service-base/tests/db/migration/postgres/001__init.sql new file mode 100644 index 0000000000..fe3714fac1 --- /dev/null +++ b/golem-component-service-base/tests/db/migration/postgres/001__init.sql @@ -0,0 +1,14 @@ +CREATE TABLE components +( + namespace text NOT NULL, + component_id uuid NOT NULL, + name text NOT NULL, + size integer NOT NULL, + version bigint NOT NULL, + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + user_component text NOT NULL, + protected_component text NOT NULL, + protector_version bigint, + metadata bytea NOT NULL, + PRIMARY KEY (namespace, component_id, version) +); diff --git a/golem-component-service-base/tests/db/migration/sqlite/001__init.sql b/golem-component-service-base/tests/db/migration/sqlite/001__init.sql new file mode 100644 index 0000000000..673b3083df --- /dev/null +++ b/golem-component-service-base/tests/db/migration/sqlite/001__init.sql @@ -0,0 +1,14 @@ +CREATE TABLE components +( + namespace text NOT NULL, + component_id uuid NOT NULL, + name text NOT NULL, + size integer NOT NULL, + version bigint NOT NULL, + created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, + user_component text NOT NULL, + protected_component text NOT NULL, + protector_version bigint, + metadata blob NOT NULL, + PRIMARY KEY (namespace, component_id, version) +); diff --git a/golem-component-service/Cargo.toml b/golem-component-service/Cargo.toml index 809d08554b..5d45aa1ce9 100644 --- a/golem-component-service/Cargo.toml +++ b/golem-component-service/Cargo.toml @@ -34,7 +34,7 @@ poem-openapi = { workspace = true } prometheus = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -sqlx = { version = "0.7", features = [ +sqlx = { workspace = true, features = [ "runtime-tokio", "sqlite", "postgres", diff --git a/golem-component-service/src/lib.rs b/golem-component-service/src/lib.rs index 88c17f2986..2302ca181b 100644 --- a/golem-component-service/src/lib.rs +++ b/golem-component-service/src/lib.rs @@ -16,5 +16,4 @@ pub mod api; pub mod config; pub mod grpcapi; pub mod metrics; -pub mod repo; pub mod service; diff --git a/golem-component-service/src/repo/component.rs b/golem-component-service/src/repo/component.rs deleted file mode 100644 index 30149a4635..0000000000 --- a/golem-component-service/src/repo/component.rs +++ /dev/null @@ -1,298 +0,0 @@ -// Copyright 2024 Golem Cloud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::ops::Deref; -use std::result::Result; -use std::sync::Arc; - -use async_trait::async_trait; -use golem_common::model::ComponentId; -use sqlx::{Database, Pool}; -use uuid::Uuid; - -use crate::repo::RepoError; -use golem_service_base::model::*; - -#[derive(sqlx::FromRow, Debug, Clone)] -pub struct ComponentRecord { - pub component_id: Uuid, - pub name: String, - pub size: i32, - pub version: i64, - pub user_component: String, - pub protected_component: String, - pub protector_version: Option, - pub metadata: String, -} - -impl From for Component { - fn from(value: ComponentRecord) -> Self { - let metadata: ComponentMetadata = serde_json::from_str(&value.metadata).unwrap(); - let versioned_component_id: VersionedComponentId = VersionedComponentId { - component_id: ComponentId(value.component_id), - version: value.version as u64, - }; - let protected_component_id: ProtectedComponentId = ProtectedComponentId { - versioned_component_id: versioned_component_id.clone(), - }; - let user_component_id: UserComponentId = UserComponentId { - versioned_component_id: versioned_component_id.clone(), - }; - Component { - component_name: ComponentName(value.name), - component_size: value.size as u64, - metadata, - versioned_component_id, - user_component_id, - protected_component_id, - } - } -} - -impl From for ComponentRecord { - fn from(value: Component) -> Self { - Self { - component_id: value.versioned_component_id.component_id.0, - name: value.component_name.0, - size: value.component_size as i32, - version: value.versioned_component_id.version as i64, - user_component: value.versioned_component_id.slug(), - protected_component: value.protected_component_id.slug(), - protector_version: None, - metadata: serde_json::to_string(&value.metadata).unwrap(), - } - } -} - -#[async_trait] -pub trait ComponentRepo { - async fn upsert(&self, component: &ComponentRecord) -> Result<(), RepoError>; - - async fn get(&self, component_id: &Uuid) -> Result, RepoError>; - - async fn get_all(&self) -> Result, RepoError>; - - async fn get_latest_version( - &self, - component_id: &Uuid, - ) -> Result, RepoError>; - - async fn get_by_version( - &self, - component_id: &Uuid, - version: u64, - ) -> Result, RepoError>; - - async fn get_by_name(&self, name: &str) -> Result, RepoError>; - - async fn delete(&self, component_id: &Uuid) -> Result<(), RepoError>; -} - -pub struct DbComponentRepo { - db_pool: Arc>, -} - -impl DbComponentRepo { - pub fn new(db_pool: Arc>) -> Self { - Self { db_pool } - } -} - -#[async_trait] -impl ComponentRepo for DbComponentRepo { - async fn upsert(&self, component: &ComponentRecord) -> Result<(), RepoError> { - sqlx::query( - r#" - INSERT INTO components - (component_id, version, name, size, user_component, protected_component, protector_version, metadata) - VALUES - ($1, $2, $3, $4, $5, $6, $7, $8::jsonb) - ON CONFLICT (component_id, version) DO UPDATE - SET name = $3, - size = $4, - user_component = $5, - protected_component = $6, - protector_version = $7, - metadata = $8::jsonb - "#, - ) - .bind(component.component_id) - .bind(component.version) - .bind(component.name.clone()) - .bind(component.size) - .bind(component.user_component.clone()) - .bind(component.protected_component.clone()) - .bind(component.protector_version) - .bind(component.metadata.clone()) - .execute(self.db_pool.deref()) - .await?; - - Ok(()) - } - - async fn get(&self, component_id: &Uuid) -> Result, RepoError> { - sqlx::query_as::<_, ComponentRecord>("SELECT component_id, version, name, size, user_component, protected_component, protector_version, CAST(metadata AS TEXT) AS metadata FROM components WHERE component_id = $1") - .bind(component_id) - .fetch_all(self.db_pool.deref()) - .await - .map_err(|e| e.into()) - } - - async fn get_all(&self) -> Result, RepoError> { - sqlx::query_as::<_, ComponentRecord>("SELECT component_id, version, name, size, user_component, protected_component, protector_version, CAST(metadata AS TEXT) AS metadata FROM components") - .fetch_all(self.db_pool.deref()) - .await - .map_err(|e| e.into()) - } - - async fn get_latest_version( - &self, - component_id: &Uuid, - ) -> Result, RepoError> { - sqlx::query_as::<_, ComponentRecord>( - "SELECT component_id, version, name, size, user_component, protected_component, protector_version, CAST(metadata AS TEXT) AS metadata FROM components WHERE component_id = $1 ORDER BY version DESC LIMIT 1", - ) - .bind(component_id) - .fetch_optional(self.db_pool.deref()) - .await - .map_err(|e| e.into()) - } - - async fn get_by_version( - &self, - component_id: &Uuid, - version: u64, - ) -> Result, RepoError> { - sqlx::query_as::<_, ComponentRecord>( - "SELECT component_id, version, name, size, user_component, protected_component, protector_version, CAST(metadata AS TEXT) AS metadata FROM components WHERE component_id = $1 AND version = $2", - ) - .bind(component_id) - .bind(version as i64) - .fetch_optional(self.db_pool.deref()) - .await - .map_err(|e| e.into()) - } - - async fn get_by_name(&self, name: &str) -> Result, RepoError> { - sqlx::query_as::<_, ComponentRecord>( - "SELECT component_id, version, name, size, user_component, protected_component, protector_version, CAST(metadata AS TEXT) AS metadata FROM components WHERE name = $1", - ) - .bind(name) - .fetch_all(self.db_pool.deref()) - .await - .map_err(|e| e.into()) - } - - async fn delete(&self, component_id: &Uuid) -> Result<(), RepoError> { - sqlx::query("DELETE FROM components WHERE component_id = $1") - .bind(component_id) - .execute(self.db_pool.deref()) - .await?; - Ok(()) - } -} - -#[async_trait] -impl ComponentRepo for DbComponentRepo { - async fn upsert(&self, component: &ComponentRecord) -> Result<(), RepoError> { - sqlx::query( - r#" - INSERT INTO components - (component_id, version, name, size, user_component, protected_component, protector_version, metadata) - VALUES - ($1, $2, $3, $4, $5, $6, $7, $8::jsonb) - ON CONFLICT (component_id, version) DO UPDATE - SET name = $3, - size = $4, - user_component = $5, - protected_component = $6, - protector_version = $7, - metadata = $8::jsonb - "#, - ) - .bind(component.component_id) - .bind(component.version) - .bind(component.name.clone()) - .bind(component.size) - .bind(component.user_component.clone()) - .bind(component.protected_component.clone()) - .bind(component.protector_version) - .bind(component.metadata.clone()) - .execute(self.db_pool.deref()) - .await?; - - Ok(()) - } - - async fn get_all(&self) -> Result, RepoError> { - sqlx::query_as::<_, ComponentRecord>("SELECT component_id, name, size, version, user_component, protected_component, protector_version, jsonb_pretty(components.metadata) AS metadata FROM components") - .fetch_all(self.db_pool.deref()) - .await - .map_err(|e| e.into()) - } - - async fn get(&self, component_id: &Uuid) -> Result, RepoError> { - sqlx::query_as::<_, ComponentRecord>("SELECT component_id, name, size, version, user_component, protected_component, protector_version, jsonb_pretty(components.metadata) AS metadata FROM components WHERE component_id = $1") - .bind(component_id) - .fetch_all(self.db_pool.deref()) - .await - .map_err(|e| e.into()) - } - - async fn get_by_name(&self, name: &str) -> Result, RepoError> { - sqlx::query_as::<_, ComponentRecord>( - "SELECT component_id, name, size, version, user_component, protected_component, protector_version, jsonb_pretty(components.metadata) AS metadata FROM components WHERE name = $1", - ) - .bind(name) - .fetch_all(self.db_pool.deref()) - .await - .map_err(|e| e.into()) - } - - async fn get_latest_version( - &self, - component_id: &Uuid, - ) -> Result, RepoError> { - sqlx::query_as::<_, ComponentRecord>( - "SELECT component_id, name, size, version, user_component, protected_component, protector_version, jsonb_pretty(components.metadata) AS metadata FROM components WHERE component_id = $1 ORDER BY version DESC LIMIT 1", - ) - .bind(component_id) - .fetch_optional(self.db_pool.deref()) - .await - .map_err(|e| e.into()) - } - - async fn get_by_version( - &self, - component_id: &Uuid, - version: u64, - ) -> Result, RepoError> { - sqlx::query_as::<_, ComponentRecord>( - "SELECT component_id, name, size, version, user_component, protected_component, protector_version, jsonb_pretty(components.metadata) AS metadata FROM components WHERE component_id = $1 AND version = $2", - ) - .bind(component_id) - .bind(version as i64) - .fetch_optional(self.db_pool.deref()) - .await - .map_err(|e| e.into()) - } - - async fn delete(&self, component_id: &Uuid) -> Result<(), RepoError> { - sqlx::query("DELETE FROM components WHERE component_id = $1") - .bind(component_id) - .execute(self.db_pool.deref()) - .await?; - Ok(()) - } -} diff --git a/golem-service-base/Cargo.toml b/golem-service-base/Cargo.toml index a8cd4d61e2..5fa53838eb 100644 --- a/golem-service-base/Cargo.toml +++ b/golem-service-base/Cargo.toml @@ -17,6 +17,7 @@ async-trait = { workspace = true } aws-config = { workspace = true } aws-sdk-s3 = { workspace = true } bigdecimal = "0.4.2" +bincode = { workspace = true } futures = { workspace = true } http = { workspace = true } http_02 = { workspace = true } @@ -26,7 +27,7 @@ poem-openapi = { workspace = true } rand = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -sqlx = { version = "0.7", features = [ +sqlx = { workspace = true, features = [ "runtime-tokio", "sqlite", "postgres", diff --git a/golem-service-base/src/model.rs b/golem-service-base/src/model.rs index 25bcfa8886..544ced33d5 100644 --- a/golem-service-base/src/model.rs +++ b/golem-service-base/src/model.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use bincode::{Decode, Encode}; use golem_common::model::{ ComponentId, ComponentVersion, ScanCursor, ShardId, Timestamp, WorkerFilter, WorkerStatus, }; @@ -163,7 +164,7 @@ impl From for golem_api_grpc::proto::golem::component::Pro #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Object)] pub struct Empty {} -#[derive(Debug, Clone, PartialEq, Eq, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Object, Encode, Decode)] pub struct TypeResult { pub ok: Option>, pub err: Option>, @@ -195,7 +196,7 @@ impl Serialize for TypeResult { } } -#[derive(Debug, Clone, PartialEq, Eq, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Object, Encode, Decode)] pub struct NameTypePair { pub name: String, pub typ: Box, @@ -225,7 +226,7 @@ impl Serialize for NameTypePair { } } -#[derive(Debug, Clone, PartialEq, Eq, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Object, Encode, Decode)] pub struct NameOptionTypePair { pub name: String, pub typ: Option>, @@ -256,7 +257,7 @@ impl Serialize for NameOptionTypePair { } } -#[derive(Debug, Clone, PartialEq, Eq, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Object, Encode, Decode)] pub struct TypeVariant { pub cases: Vec, } @@ -280,7 +281,7 @@ impl Serialize for TypeVariant { } } -#[derive(Debug, Clone, PartialEq, Eq, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Object, Encode, Decode)] pub struct TypeOption { pub inner: Box, } @@ -304,7 +305,7 @@ impl Serialize for TypeOption { } } -#[derive(Debug, Clone, PartialEq, Eq, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Object, Encode, Decode)] pub struct TypeEnum { pub cases: Vec, } @@ -328,7 +329,7 @@ impl Serialize for TypeEnum { } } -#[derive(Debug, Clone, PartialEq, Eq, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Object, Encode, Decode)] pub struct TypeFlags { pub cases: Vec, } @@ -352,7 +353,7 @@ impl Serialize for TypeFlags { } } -#[derive(Debug, Clone, PartialEq, Eq, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Object, Encode, Decode)] pub struct TypeRecord { pub cases: Vec, } @@ -376,7 +377,7 @@ impl Serialize for TypeRecord { } } -#[derive(Debug, Clone, PartialEq, Eq, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Object, Encode, Decode)] pub struct TypeTuple { pub items: Vec, } @@ -400,7 +401,7 @@ impl Serialize for TypeTuple { } } -#[derive(Debug, Clone, PartialEq, Eq, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Object, Encode, Decode)] pub struct TypeList { pub inner: Box, } @@ -424,7 +425,7 @@ impl Serialize for TypeList { } } -#[derive(Debug, Clone, PartialEq, Eq, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Object, Encode, Decode)] pub struct TypeStr; impl<'de> Deserialize<'de> for TypeStr { @@ -454,7 +455,7 @@ impl Serialize for TypeStr { } } -#[derive(Debug, Clone, PartialEq, Eq, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Object, Encode, Decode)] pub struct TypeChr; impl<'de> Deserialize<'de> for TypeChr { @@ -484,7 +485,7 @@ impl Serialize for TypeChr { } } -#[derive(Debug, Clone, PartialEq, Eq, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Object, Encode, Decode)] pub struct TypeF64; impl<'de> Deserialize<'de> for TypeF64 { @@ -514,7 +515,7 @@ impl Serialize for TypeF64 { } } -#[derive(Debug, Clone, PartialEq, Eq, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Object, Encode, Decode)] pub struct TypeF32; impl<'de> Deserialize<'de> for TypeF32 { @@ -544,7 +545,7 @@ impl Serialize for TypeF32 { } } -#[derive(Debug, Clone, PartialEq, Eq, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Object, Encode, Decode)] pub struct TypeU64; impl<'de> Deserialize<'de> for TypeU64 { @@ -574,7 +575,7 @@ impl Serialize for TypeU64 { } } -#[derive(Debug, Clone, PartialEq, Eq, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Object, Encode, Decode)] pub struct TypeS64; impl<'de> Deserialize<'de> for TypeS64 { @@ -604,7 +605,7 @@ impl Serialize for TypeS64 { } } -#[derive(Debug, Clone, PartialEq, Eq, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Object, Encode, Decode)] pub struct TypeU32; impl<'de> Deserialize<'de> for TypeU32 { @@ -634,7 +635,7 @@ impl Serialize for TypeU32 { } } -#[derive(Debug, Clone, PartialEq, Eq, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Object, Encode, Decode)] pub struct TypeS32; impl<'de> Deserialize<'de> for TypeS32 { @@ -664,7 +665,7 @@ impl Serialize for TypeS32 { } } -#[derive(Debug, Clone, PartialEq, Eq, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Object, Encode, Decode)] pub struct TypeU16; impl<'de> Deserialize<'de> for TypeU16 { @@ -694,7 +695,7 @@ impl Serialize for TypeU16 { } } -#[derive(Debug, Clone, PartialEq, Eq, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Object, Encode, Decode)] pub struct TypeS16; impl<'de> Deserialize<'de> for TypeS16 { @@ -724,7 +725,7 @@ impl Serialize for TypeS16 { } } -#[derive(Debug, Clone, PartialEq, Eq, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Object, Encode, Decode)] pub struct TypeU8; impl<'de> Deserialize<'de> for TypeU8 { @@ -754,7 +755,7 @@ impl Serialize for TypeU8 { } } -#[derive(Debug, Clone, PartialEq, Eq, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Object, Encode, Decode)] pub struct TypeS8; impl<'de> Deserialize<'de> for TypeS8 { @@ -784,7 +785,7 @@ impl Serialize for TypeS8 { } } -#[derive(Debug, Clone, PartialEq, Eq, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Object, Encode, Decode)] pub struct TypeBool; impl<'de> Deserialize<'de> for TypeBool { @@ -814,7 +815,7 @@ impl Serialize for TypeBool { } } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Enum)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Enum, Encode, Decode)] pub enum ResourceMode { Borrowed, Owned, @@ -838,7 +839,7 @@ impl From for AnalysedResourceMode { } } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Object, Encode, Decode)] pub struct TypeHandle { resource_id: u64, mode: ResourceMode, @@ -871,7 +872,7 @@ impl From for golem_wasm_rpc::protobuf::TypeHandle { } } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Union)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Union, Encode, Decode)] #[oai(discriminator_name = "type", one_of = true)] pub enum Type { Variant(TypeVariant), @@ -1168,7 +1169,7 @@ impl From for golem_wasm_rpc::protobuf::Type { } } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Object, Encode, Decode)] pub struct FunctionParameter { pub name: String, // TODO: Fix this in DB. Temp fix for now. @@ -1198,7 +1199,7 @@ impl From for golem_api_grpc::proto::golem::component::Functi } } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Object, Encode, Decode)] pub struct FunctionResult { pub name: Option, // TODO: Fix this in DB. Temp fix for now. @@ -1228,7 +1229,7 @@ impl From for golem_api_grpc::proto::golem::component::FunctionR } } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Object, Encode, Decode)] pub struct ExportInstance { pub name: String, pub functions: Vec, @@ -1264,7 +1265,7 @@ impl From for golem_api_grpc::proto::golem::component::ExportIns } } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Object, Encode, Decode)] pub struct ExportFunction { pub name: String, pub parameters: Vec, @@ -1311,7 +1312,7 @@ impl From for golem_api_grpc::proto::golem::component::ExportFun } } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Union)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Union, Encode, Decode)] #[oai(discriminator_name = "type", one_of = true)] pub enum Export { Instance(ExportInstance), @@ -1370,7 +1371,20 @@ impl From for golem_api_grpc::proto::golem::component::Export { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize, Deserialize, Object)] +#[derive( + Debug, + Clone, + PartialEq, + Eq, + Hash, + Ord, + PartialOrd, + Serialize, + Deserialize, + Object, + Encode, + Decode, +)] pub struct VersionedName { pub name: String, pub version: String, @@ -1394,7 +1408,20 @@ impl From for golem_api_grpc::proto::golem::component::VersionedN } } -#[derive(Debug, Clone, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize, Deserialize, Object)] +#[derive( + Debug, + Clone, + PartialEq, + Eq, + Hash, + Ord, + PartialOrd, + Serialize, + Deserialize, + Object, + Encode, + Decode, +)] pub struct ProducerField { pub name: String, pub values: Vec, @@ -1418,7 +1445,20 @@ impl From for golem_api_grpc::proto::golem::component::ProducerFi } } -#[derive(Debug, Clone, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize, Deserialize, Object)] +#[derive( + Debug, + Clone, + PartialEq, + Eq, + Hash, + Ord, + PartialOrd, + Serialize, + Deserialize, + Object, + Encode, + Decode, +)] pub struct Producers { pub fields: Vec, } @@ -1716,7 +1756,7 @@ impl From for golem_wasm_ast::analysis::AnalysedType { } } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Object, Encode, Decode)] pub struct LinearMemory { /// Initial size of the linear memory in bytes pub initial: u64, @@ -1759,7 +1799,7 @@ impl From for golem_api_grpc::proto::golem::component::LinearMemor } } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Object)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Object, Encode, Decode)] pub struct ComponentMetadata { pub exports: Vec, pub producers: Vec, diff --git a/golem-worker-service-base/Cargo.toml b/golem-worker-service-base/Cargo.toml index d6d832968e..a36bbdc723 100644 --- a/golem-worker-service-base/Cargo.toml +++ b/golem-worker-service-base/Cargo.toml @@ -45,7 +45,7 @@ serde_json = { workspace = true } serde_yaml = { workspace = true } strum = { workspace = true } strum_macros = { workspace = true } -sqlx = { version = "0.7", features = [ +sqlx = { workspace = true, features = [ "runtime-tokio", "sqlite", "postgres", diff --git a/golem-worker-service-base/src/repo/api_definition.rs b/golem-worker-service-base/src/repo/api_definition.rs index 3ae07ef1f6..cbd31d261d 100644 --- a/golem-worker-service-base/src/repo/api_definition.rs +++ b/golem-worker-service-base/src/repo/api_definition.rs @@ -1,3 +1,17 @@ +// Copyright 2024 Golem Cloud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + use crate::api_definition::http::HttpApiDefinition; use crate::repo::RepoError; use async_trait::async_trait; diff --git a/golem-worker-service-base/src/repo/api_deployment.rs b/golem-worker-service-base/src/repo/api_deployment.rs index 25018dfba1..dbd2ccb923 100644 --- a/golem-worker-service-base/src/repo/api_deployment.rs +++ b/golem-worker-service-base/src/repo/api_deployment.rs @@ -1,3 +1,17 @@ +// Copyright 2024 Golem Cloud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + use crate::api_definition::ApiSite; use crate::repo::api_definition::ApiDefinitionRecord; use crate::repo::RepoError; diff --git a/golem-worker-service-base/src/repo/mod.rs b/golem-worker-service-base/src/repo/mod.rs index 94985de706..edba60ef2d 100644 --- a/golem-worker-service-base/src/repo/mod.rs +++ b/golem-worker-service-base/src/repo/mod.rs @@ -1,3 +1,17 @@ +// Copyright 2024 Golem Cloud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + pub mod api_definition; pub mod api_deployment; diff --git a/golem-worker-service-base/src/service/api_definition.rs b/golem-worker-service-base/src/service/api_definition.rs index 25a54e2b3b..d1be5597b2 100644 --- a/golem-worker-service-base/src/service/api_definition.rs +++ b/golem-worker-service-base/src/service/api_definition.rs @@ -1,3 +1,17 @@ +// Copyright 2024 Golem Cloud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + use std::fmt::{Debug, Display}; use std::hash::Hash; use std::sync::Arc; diff --git a/golem-worker-service-base/src/service/api_definition_lookup.rs b/golem-worker-service-base/src/service/api_definition_lookup.rs index daade0218e..db9873e6ff 100644 --- a/golem-worker-service-base/src/service/api_definition_lookup.rs +++ b/golem-worker-service-base/src/service/api_definition_lookup.rs @@ -1,3 +1,17 @@ +// Copyright 2024 Golem Cloud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + use std::fmt::Display; use std::sync::Arc; diff --git a/golem-worker-service-base/src/service/api_definition_validator.rs b/golem-worker-service-base/src/service/api_definition_validator.rs index 415b10d954..58eb4885f6 100644 --- a/golem-worker-service-base/src/service/api_definition_validator.rs +++ b/golem-worker-service-base/src/service/api_definition_validator.rs @@ -1,3 +1,17 @@ +// Copyright 2024 Golem Cloud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + use async_trait::async_trait; use serde::{Deserialize, Serialize}; diff --git a/golem-worker-service-base/src/service/api_deployment.rs b/golem-worker-service-base/src/service/api_deployment.rs index 83723f3de0..6c56d8c303 100644 --- a/golem-worker-service-base/src/service/api_deployment.rs +++ b/golem-worker-service-base/src/service/api_deployment.rs @@ -1,3 +1,17 @@ +// Copyright 2024 Golem Cloud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + use crate::api_definition::{ApiDefinitionId, ApiDeployment, ApiSite, ApiSiteString}; use std::collections::HashSet; diff --git a/golem-worker-service-base/src/service/mod.rs b/golem-worker-service-base/src/service/mod.rs index 65f7185835..021a555368 100644 --- a/golem-worker-service-base/src/service/mod.rs +++ b/golem-worker-service-base/src/service/mod.rs @@ -1,5 +1,18 @@ -pub mod api_definition; +// Copyright 2024 Golem Cloud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +pub mod api_definition; pub mod api_definition_lookup; pub mod api_definition_validator; pub mod api_deployment; From db7b9274ea5a007aff719dbad6348db37dab2a10 Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Fri, 28 Jun 2024 22:16:33 +0200 Subject: [PATCH 02/30] component service --- golem-component-service-base/Cargo.toml | 5 + .../src/repo/component.rs | 9 + .../src/service/component.rs | 179 +++++++++++++----- .../src/service/mod.rs | 1 + golem-component-service/src/api/component.rs | 30 ++- .../src/grpcapi/component.rs | 37 +++- golem-component-service/src/service.rs | 18 +- golem-service-base/src/auth.rs | 43 +++++ golem-service-base/src/lib.rs | 1 - 9 files changed, 246 insertions(+), 77 deletions(-) rename {golem-component-service => golem-component-service-base}/src/service/component.rs (75%) create mode 100644 golem-service-base/src/auth.rs diff --git a/golem-component-service-base/Cargo.toml b/golem-component-service-base/Cargo.toml index dfa783b758..f8b06e94ec 100644 --- a/golem-component-service-base/Cargo.toml +++ b/golem-component-service-base/Cargo.toml @@ -11,6 +11,7 @@ golem-common = { path = "../golem-common", version = "0.0.0" } golem-service-base = { path = "../golem-service-base" } golem-wasm-ast = { workspace = true } +anyhow = { workspace = true } async-trait = { workspace = true } bincode = { workspace = true } bytes = { workspace = true } @@ -24,7 +25,11 @@ sqlx = { workspace = true, features = [ "migrate", "chrono", ] } +tap = { workspace = true } thiserror = { workspace = true } tonic = { workspace = true } +tokio = { workspace = true } +tokio-stream = { workspace = true } +tokio-util = { workspace = true } tracing = { workspace = true } uuid = { workspace = true } \ No newline at end of file diff --git a/golem-component-service-base/src/repo/component.rs b/golem-component-service-base/src/repo/component.rs index 08196f1795..b3719c152e 100644 --- a/golem-component-service-base/src/repo/component.rs +++ b/golem-component-service-base/src/repo/component.rs @@ -63,6 +63,15 @@ impl TryFrom for Component { } } +impl From for VersionedComponentId { + fn from(value: ComponentRecord) -> Self { + VersionedComponentId { + component_id: ComponentId(value.component_id), + version: value.version as u64, + } + } +} + impl ComponentRecord { pub fn new( namespace: Namespace, diff --git a/golem-component-service/src/service/component.rs b/golem-component-service-base/src/service/component.rs similarity index 75% rename from golem-component-service/src/service/component.rs rename to golem-component-service-base/src/service/component.rs index 9c77ae85c3..5b3b167c1b 100644 --- a/golem-component-service/src/service/component.rs +++ b/golem-component-service-base/src/service/component.rs @@ -15,19 +15,17 @@ use std::fmt::Display; use std::sync::Arc; +use crate::service::component_compilation::ComponentCompilationService; +use crate::service::component_processor::{process_component, ComponentProcessingError}; use async_trait::async_trait; use golem_common::model::ComponentId; -use golem_component_service_base::service::component_compilation::ComponentCompilationService; -use golem_component_service_base::service::component_processor::{ - process_component, ComponentProcessingError, -}; use tap::TapFallible; use tracing::{error, info}; -use crate::repo::component::ComponentRepo; +use crate::repo::component::{ComponentRecord, ComponentRepo}; use crate::repo::RepoError; -use crate::service::component_object_store::ComponentObjectStore; use golem_service_base::model::*; +use golem_service_base::service::component_object_store::ComponentObjectStore; use golem_service_base::stream::ByteStream; #[derive(Debug, thiserror::Error)] @@ -61,53 +59,65 @@ impl From for ComponentError { } #[async_trait] -pub trait ComponentService { +pub trait ComponentService { async fn create( &self, component_name: &ComponentName, data: Vec, + namespace: &Namespace, ) -> Result; async fn update( &self, component_id: &ComponentId, data: Vec, + namespace: &Namespace, ) -> Result; async fn download( &self, component_id: &ComponentId, version: Option, + namespace: &Namespace, ) -> Result, ComponentError>; async fn download_stream( &self, component_id: &ComponentId, version: Option, + namespace: &Namespace, ) -> Result; async fn get_protected_data( &self, component_id: &ComponentId, version: Option, + namespace: &Namespace, ) -> Result>, ComponentError>; async fn find_by_name( &self, component_name: Option, + namespace: &Namespace, ) -> Result, ComponentError>; async fn get_by_version( &self, component_id: &VersionedComponentId, + namespace: &Namespace, ) -> Result, ComponentError>; async fn get_latest_version( &self, component_id: &ComponentId, + _namespace: &Namespace, ) -> Result, ComponentError>; - async fn get(&self, component_id: &ComponentId) -> Result, ComponentError>; + async fn get( + &self, + component_id: &ComponentId, + namespace: &Namespace, + ) -> Result, ComponentError>; } pub struct ComponentServiceDefault { @@ -131,16 +141,19 @@ impl ComponentServiceDefault { } #[async_trait] -impl ComponentService for ComponentServiceDefault { +impl + Eq + Clone + Send + Sync> ComponentService + for ComponentServiceDefault +{ async fn create( &self, component_name: &ComponentName, data: Vec, + namespace: &Namespace, ) -> Result { let tn = component_name.0.clone(); info!("Creating component with name {}", tn); - self.check_new_name(component_name).await?; + self.check_new_name(component_name, namespace).await?; let metadata = process_component(&data)?; @@ -182,9 +195,10 @@ impl ComponentService for ComponentServiceDefault { protected_component_id, }; - self.component_repo - .upsert(&component.clone().into()) - .await?; + let record = ComponentRecord::new(namespace, component.clone()) + .map_err(|e| ComponentError::internal(e, "Failed to convert record"))?; + + self.component_repo.upsert(&record).await?; self.component_compilation .enqueue_compilation(&component.versioned_component_id.component_id, 0) @@ -197,6 +211,7 @@ impl ComponentService for ComponentServiceDefault { &self, component_id: &ComponentId, data: Vec, + namespace: &Namespace, ) -> Result { info!("Updating component {}", component_id); @@ -204,11 +219,14 @@ impl ComponentService for ComponentServiceDefault { let next_component = self .component_repo - .get_latest_version(&component_id.0) + .get_latest_version(namespace.to_string().as_str(), &component_id.0) .await? - .map(Component::from) - .map(Component::next_version) - .ok_or(ComponentError::UnknownComponentId(component_id.clone()))?; + .ok_or(ComponentError::UnknownComponentId(component_id.clone())) + .and_then(|c| { + c.try_into() + .map_err(|e| ComponentError::internal(e, "Failed to convert record")) + }) + .map(Component::next_version)?; info!( "Uploaded component {} version {} with exports {:?}", @@ -230,10 +248,10 @@ impl ComponentService for ComponentServiceDefault { metadata, ..next_component }; + let record = ComponentRecord::new(namespace, component.clone()) + .map_err(|e| ComponentError::internal(e, "Failed to convert record"))?; - self.component_repo - .upsert(&component.clone().into()) - .await?; + self.component_repo.upsert(&record).await?; self.component_compilation .enqueue_compilation(component_id, component.versioned_component_id.version) @@ -246,6 +264,7 @@ impl ComponentService for ComponentServiceDefault { &self, component_id: &ComponentId, version: Option, + namespace: &Namespace, ) -> Result, ComponentError> { let versioned_component_id = { match version { @@ -255,10 +274,9 @@ impl ComponentService for ComponentServiceDefault { }, None => self .component_repo - .get_latest_version(&component_id.0) + .get_latest_version(namespace.to_string().as_str(), &component_id.0) .await? - .map(Component::from) - .map(|t| t.versioned_component_id) + .map(|c| c.into()) .ok_or(ComponentError::UnknownComponentId(component_id.clone()))?, } }; @@ -282,6 +300,7 @@ impl ComponentService for ComponentServiceDefault { &self, component_id: &ComponentId, version: Option, + namespace: &Namespace, ) -> Result { let versioned_component_id = { match version { @@ -291,10 +310,9 @@ impl ComponentService for ComponentServiceDefault { }, None => self .component_repo - .get_latest_version(&component_id.0) + .get_latest_version(namespace.to_string().as_str(), &component_id.0) .await? - .map(Component::from) - .map(|t| t.versioned_component_id) + .map(|c| c.into()) .ok_or(ComponentError::UnknownComponentId(component_id.clone()))?, } }; @@ -319,6 +337,7 @@ impl ComponentService for ComponentServiceDefault { &self, component_id: &ComponentId, version: Option, + namespace: &Namespace, ) -> Result>, ComponentError> { info!( "Getting component {} version {} data", @@ -328,7 +347,7 @@ impl ComponentService for ComponentServiceDefault { let latest_component = self .component_repo - .get_latest_version(&component_id.0) + .get_latest_version(namespace.to_string().as_str(), &component_id.0) .await?; let v = match latest_component { @@ -368,28 +387,53 @@ impl ComponentService for ComponentServiceDefault { async fn find_by_name( &self, component_name: Option, + namespace: &Namespace, ) -> Result, ComponentError> { let tn = component_name.clone().map_or("N/A".to_string(), |n| n.0); info!("Getting component name {}", tn); - let result = match component_name { - Some(name) => self.component_repo.get_by_name(&name.0).await?, + let records = match component_name { + Some(name) => { + self.component_repo + .get_by_name(namespace.to_string().as_str(), &name.0) + .await? + } None => self.component_repo.get_all().await?, }; - Ok(result.into_iter().map(|t| t.into()).collect()) + let values: Vec = records + .iter() + .map(|d| d.clone().try_into()) + .collect::, _>>() + .map_err(|e| ComponentError::internal(e, "Failed to convert record".to_string()))?; + + Ok(values) } - async fn get(&self, component_id: &ComponentId) -> Result, ComponentError> { + async fn get( + &self, + component_id: &ComponentId, + namespace: &Namespace, + ) -> Result, ComponentError> { info!("Getting component {}", component_id); - let result = self.component_repo.get(&component_id.0).await?; + let records = self + .component_repo + .get(namespace.to_string().as_str(), &component_id.0) + .await?; - Ok(result.into_iter().map(|t| t.into()).collect()) + let values: Vec = records + .iter() + .map(|d| d.clone().try_into()) + .collect::, _>>() + .map_err(|e| ComponentError::internal(e, "Failed to convert record".to_string()))?; + + Ok(values) } async fn get_by_version( &self, component_id: &VersionedComponentId, + namespace: &Namespace, ) -> Result, ComponentError> { info!( "Getting component {} version {}", @@ -398,41 +442,62 @@ impl ComponentService for ComponentServiceDefault { let result = self .component_repo - .get_by_version(&component_id.component_id.0, component_id.version) + .get_by_version( + namespace.to_string().as_str(), + &component_id.component_id.0, + component_id.version, + ) .await?; - Ok(result.map(|t| t.into())) + + match result { + Some(c) => { + let value = c.try_into().map_err(|e| { + ComponentError::internal(e, "Failed to convert record".to_string()) + })?; + Ok(Some(value)) + } + None => Ok(None), + } } async fn get_latest_version( &self, component_id: &ComponentId, + namespace: &Namespace, ) -> Result, ComponentError> { info!("Getting component {} latest version", component_id); let result = self .component_repo - .get_latest_version(&component_id.0) + .get_latest_version(namespace.to_string().as_str(), &component_id.0) .await?; - Ok(result.map(|t| t.into())) + + match result { + Some(c) => { + let value = c.try_into().map_err(|e| { + ComponentError::internal(e, "Failed to convert record".to_string()) + })?; + Ok(Some(value)) + } + None => Ok(None), + } } } impl ComponentServiceDefault { - async fn check_new_name(&self, component_name: &ComponentName) -> Result<(), ComponentError> { + async fn check_new_name( + &self, + component_name: &ComponentName, + namespace: &Namespace, + ) -> Result<(), ComponentError> { let existing_components = self .component_repo - .get_by_name(&component_name.0) + .get_by_name(namespace.to_string().as_str(), &component_name.0) .await .tap_err(|e| error!("Error getting existing components: {}", e))?; - existing_components - .into_iter() - .next() - .map(Component::from) - .map_or(Ok(()), |t| { - Err(ComponentError::AlreadyExists( - t.versioned_component_id.component_id, - )) - }) + existing_components.into_iter().next().map_or(Ok(()), |c| { + Err(ComponentError::AlreadyExists(ComponentId(c.component_id))) + }) } fn get_user_object_store_key(&self, id: &UserComponentId) -> String { @@ -482,11 +547,14 @@ impl ComponentServiceDefault { pub struct ComponentServiceNoop {} #[async_trait] -impl ComponentService for ComponentServiceNoop { +impl + Eq + Clone + Send + Sync> ComponentService + for ComponentServiceNoop +{ async fn create( &self, _component_name: &ComponentName, _data: Vec, + _namespace: &Namespace, ) -> Result { let fake_component = Component { component_name: ComponentName("fake".to_string()), @@ -521,6 +589,7 @@ impl ComponentService for ComponentServiceNoop { &self, _component_id: &ComponentId, _data: Vec, + _namespace: &Namespace, ) -> Result { let fake_component = Component { component_name: ComponentName("fake".to_string()), @@ -555,6 +624,7 @@ impl ComponentService for ComponentServiceNoop { &self, _component_id: &ComponentId, _version: Option, + _namespace: &Namespace, ) -> Result, ComponentError> { Ok(vec![]) } @@ -563,6 +633,7 @@ impl ComponentService for ComponentServiceNoop { &self, _component_id: &ComponentId, _version: Option, + _namespace: &Namespace, ) -> Result { Ok(ByteStream::empty()) } @@ -571,6 +642,7 @@ impl ComponentService for ComponentServiceNoop { &self, _component_id: &ComponentId, _version: Option, + _namespace: &Namespace, ) -> Result>, ComponentError> { Ok(None) } @@ -578,6 +650,7 @@ impl ComponentService for ComponentServiceNoop { async fn find_by_name( &self, _component_name: Option, + _namespace: &Namespace, ) -> Result, ComponentError> { Ok(vec![]) } @@ -585,6 +658,7 @@ impl ComponentService for ComponentServiceNoop { async fn get_by_version( &self, _component_id: &VersionedComponentId, + _namespace: &Namespace, ) -> Result, ComponentError> { Ok(None) } @@ -592,11 +666,16 @@ impl ComponentService for ComponentServiceNoop { async fn get_latest_version( &self, _component_id: &ComponentId, + _namespace: &Namespace, ) -> Result, ComponentError> { Ok(None) } - async fn get(&self, _component_id: &ComponentId) -> Result, ComponentError> { + async fn get( + &self, + _component_id: &ComponentId, + _namespace: &Namespace, + ) -> Result, ComponentError> { Ok(vec![]) } } diff --git a/golem-component-service-base/src/service/mod.rs b/golem-component-service-base/src/service/mod.rs index 1dc64f4ee0..42e8c06ac1 100644 --- a/golem-component-service-base/src/service/mod.rs +++ b/golem-component-service-base/src/service/mod.rs @@ -12,5 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub mod component; pub mod component_compilation; pub mod component_processor; diff --git a/golem-component-service/src/api/component.rs b/golem-component-service/src/api/component.rs index 45adf118cc..55683a0dc6 100644 --- a/golem-component-service/src/api/component.rs +++ b/golem-component-service/src/api/component.rs @@ -15,9 +15,12 @@ use futures_util::TryStreamExt; use std::sync::Arc; -use crate::service::component::{ComponentError as ComponentServiceError, ComponentService}; use golem_common::model::ComponentId; +use golem_component_service_base::service::component::{ + ComponentError as ComponentServiceError, ComponentService, +}; use golem_service_base::api_tags::ApiTags; +use golem_service_base::auth::DefaultNamespace; use golem_service_base::model::*; use poem::error::ReadBodyError; use poem::Body; @@ -95,7 +98,7 @@ impl From for ComponentError { } pub struct ComponentApi { - pub component_service: Arc, + pub component_service: Arc + Sync + Send>, } #[OpenApi(prefix_path = "/v2/components", tag = ApiTags::Component)] @@ -104,7 +107,10 @@ impl ComponentApi { async fn create_component(&self, payload: UploadPayload) -> Result> { let data = payload.component.into_vec().await?; let component_name = payload.name; - let response = self.component_service.create(&component_name, data).await?; + let response = self + .component_service + .create(&component_name, data, &DefaultNamespace::default()) + .await?; Ok(Json(response)) } @@ -119,7 +125,10 @@ impl ComponentApi { wasm: Binary, ) -> Result> { let data = wasm.0.into_vec().await?; - let response = self.component_service.update(&component_id.0, data).await?; + let response = self + .component_service + .update(&component_id.0, data, &DefaultNamespace::default()) + .await?; Ok(Json(response)) } @@ -135,7 +144,7 @@ impl ComponentApi { ) -> Result> { let bytes = self .component_service - .download_stream(&component_id.0, version.0) + .download_stream(&component_id.0, version.0, &DefaultNamespace::default()) .await?; Ok(Binary(Body::from_bytes_stream(bytes.map_err(|e| { std::io::Error::new(std::io::ErrorKind::Other, e.to_string()) @@ -151,7 +160,10 @@ impl ComponentApi { &self, component_id: Path, ) -> Result>> { - let response = self.component_service.get(&component_id.0).await?; + let response = self + .component_service + .get(&component_id.0, &DefaultNamespace::default()) + .await?; Ok(Json(response)) } @@ -181,7 +193,7 @@ impl ComponentApi { let response = self .component_service - .get_by_version(&versioned_component_id) + .get_by_version(&versioned_component_id, &DefaultNamespace::default()) .await?; match response { @@ -203,7 +215,7 @@ impl ComponentApi { ) -> Result> { let response = self .component_service - .get_latest_version(&component_id.0) + .get_latest_version(&component_id.0, &DefaultNamespace::default()) .await?; match response { @@ -221,7 +233,7 @@ impl ComponentApi { ) -> Result>> { let response = self .component_service - .find_by_name(component_name.0) + .find_by_name(component_name.0, &DefaultNamespace::default()) .await?; Ok(Json(response)) diff --git a/golem-component-service/src/grpcapi/component.rs b/golem-component-service/src/grpcapi/component.rs index 5b38c34687..59a0de0de0 100644 --- a/golem-component-service/src/grpcapi/component.rs +++ b/golem-component-service/src/grpcapi/component.rs @@ -35,7 +35,8 @@ use golem_common::model::ComponentId; use golem_service_base::stream::ByteStream; use tonic::{Request, Response, Status, Streaming}; -use crate::service::component; +use golem_component_service_base::service::component; +use golem_service_base::auth::DefaultNamespace; impl From for ComponentError { fn from(value: component::ComponentError) -> Self { @@ -83,7 +84,7 @@ fn internal_error(error: &str) -> ComponentError { } pub struct ComponentGrpcApi { - pub component_service: Arc, + pub component_service: Arc + Sync + Send>, } impl ComponentGrpcApi { @@ -92,7 +93,10 @@ impl ComponentGrpcApi { .component_id .and_then(|id| id.try_into().ok()) .ok_or_else(|| bad_request_error("Missing component id"))?; - let result = self.component_service.get(&id).await?; + let result = self + .component_service + .get(&id, &DefaultNamespace::default()) + .await?; Ok(result.into_iter().map(|p| p.into()).collect()) } @@ -114,7 +118,7 @@ impl ComponentGrpcApi { let result = self .component_service - .get_by_version(&versioned_component_id) + .get_by_version(&versioned_component_id, &DefaultNamespace::default()) .await?; Ok(result.map(|p| p.into())) } @@ -126,7 +130,10 @@ impl ComponentGrpcApi { let name: Option = request .component_name .map(golem_service_base::model::ComponentName); - let result = self.component_service.find_by_name(name).await?; + let result = self + .component_service + .find_by_name(name, &DefaultNamespace::default()) + .await?; Ok(result.into_iter().map(|p| p.into()).collect()) } @@ -138,7 +145,10 @@ impl ComponentGrpcApi { .component_id .and_then(|id| id.try_into().ok()) .ok_or_else(|| bad_request_error("Missing component id"))?; - let result = self.component_service.get_latest_version(&id).await?; + let result = self + .component_service + .get_latest_version(&id, &DefaultNamespace::default()) + .await?; match result { Some(component) => Ok(component.into()), None => Err(ComponentError { @@ -158,7 +168,10 @@ impl ComponentGrpcApi { .and_then(|id| id.try_into().ok()) .ok_or_else(|| bad_request_error("Missing component id"))?; let version = request.version; - let result = self.component_service.download_stream(&id, version).await?; + let result = self + .component_service + .download_stream(&id, version, &DefaultNamespace::default()) + .await?; Ok(result) } @@ -168,7 +181,10 @@ impl ComponentGrpcApi { data: Vec, ) -> Result { let name = golem_service_base::model::ComponentName(request.component_name); - let result = self.component_service.create(&name, data).await?; + let result = self + .component_service + .create(&name, data, &DefaultNamespace::default()) + .await?; Ok(result.into()) } @@ -181,7 +197,10 @@ impl ComponentGrpcApi { .component_id .and_then(|id| id.try_into().ok()) .ok_or_else(|| bad_request_error("Missing component id"))?; - let result = self.component_service.update(&id, data).await?; + let result = self + .component_service + .update(&id, data, &DefaultNamespace::default()) + .await?; Ok(result.into()) } } diff --git a/golem-component-service/src/service.rs b/golem-component-service/src/service.rs index 743ea9ee0d..8d400dda88 100644 --- a/golem-component-service/src/service.rs +++ b/golem-component-service/src/service.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub mod component; - use golem_component_service_base::config::ComponentCompilationConfig; use golem_component_service_base::service::component_compilation::{ ComponentCompilationService, ComponentCompilationServiceDefault, @@ -25,11 +23,15 @@ use golem_service_base::service::component_object_store; use std::sync::Arc; use crate::config::ComponentServiceConfig; -use crate::repo::component::{ComponentRepo, DbComponentRepo}; +use golem_component_service_base::repo::component::{ComponentRepo, DbComponentRepo}; +use golem_component_service_base::service::component::{ + ComponentService, ComponentServiceDefault, ComponentServiceNoop, +}; +use golem_service_base::auth::DefaultNamespace; #[derive(Clone)] pub struct Services { - pub component_service: Arc, + pub component_service: Arc + Sync + Send>, pub compilation_service: Arc, } @@ -70,8 +72,8 @@ impl Services { } }; - let component_service: Arc = - Arc::new(component::ComponentServiceDefault::new( + let component_service: Arc + Sync + Send> = + Arc::new(ComponentServiceDefault::new( component_repo.clone(), object_store.clone(), compilation_service.clone(), @@ -84,8 +86,8 @@ impl Services { } pub fn noop() -> Self { - let component_service: Arc = - Arc::new(component::ComponentServiceNoop::default()); + let component_service: Arc + Sync + Send> = + Arc::new(ComponentServiceNoop::default()); let compilation_service: Arc = Arc::new(ComponentCompilationServiceDisabled); diff --git a/golem-service-base/src/auth.rs b/golem-service-base/src/auth.rs new file mode 100644 index 0000000000..429f2dc883 --- /dev/null +++ b/golem-service-base/src/auth.rs @@ -0,0 +1,43 @@ +use std::fmt::{Display, Formatter}; + +use serde::Deserialize; + +#[derive(Default, Debug, Clone, PartialEq, Eq, Hash)] +pub struct EmptyAuthCtx(); + +impl Display for EmptyAuthCtx { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "EmptyAuthCtx") + } +} + +impl IntoIterator for EmptyAuthCtx { + type Item = (String, String); + type IntoIter = std::iter::Empty; + + fn into_iter(self) -> Self::IntoIter { + std::iter::empty() + } +} + +#[derive( + Default, Debug, Clone, PartialEq, Eq, Hash, bincode::Encode, bincode::Decode, Deserialize, +)] +pub struct DefaultNamespace(); + +impl Display for DefaultNamespace { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "default") + } +} + +impl TryFrom for DefaultNamespace { + type Error = String; + fn try_from(value: String) -> Result { + if value.as_str() == "default" { + Ok(DefaultNamespace::default()) + } else { + Err("Failed to parse empty namespace".to_string()) + } + } +} diff --git a/golem-service-base/src/lib.rs b/golem-service-base/src/lib.rs index b5ebc6ab20..29dde3dbef 100644 --- a/golem-service-base/src/lib.rs +++ b/golem-service-base/src/lib.rs @@ -18,7 +18,6 @@ pub mod model; pub mod routing_table; pub mod service; pub mod typechecker; - pub mod db; pub mod stream; pub mod type_inference; From 0125f5d5cf4b83ada23e5f2a73949654b1dda6bc Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Fri, 28 Jun 2024 22:21:26 +0200 Subject: [PATCH 03/30] service base auth --- golem-worker-service-base/src/auth.rs | 43 ------------------- golem-worker-service-base/src/lib.rs | 1 - .../tests/services_tests.rs | 2 +- .../src/api/api_definition.rs | 2 +- .../src/api/api_deployment.rs | 2 +- golem-worker-service/src/api/worker.rs | 2 +- .../src/api/worker_connect.rs | 2 +- .../src/grpcapi/api_definition.rs | 3 +- golem-worker-service/src/grpcapi/worker.rs | 2 +- golem-worker-service/src/service/component.rs | 2 +- golem-worker-service/src/service/mod.rs | 2 +- golem-worker-service/src/service/worker.rs | 2 +- .../src/worker_bridge_request_executor.rs | 4 +- .../src/worker_component_metadata_fetcher.rs | 2 +- 14 files changed, 13 insertions(+), 58 deletions(-) delete mode 100644 golem-worker-service-base/src/auth.rs diff --git a/golem-worker-service-base/src/auth.rs b/golem-worker-service-base/src/auth.rs deleted file mode 100644 index 429f2dc883..0000000000 --- a/golem-worker-service-base/src/auth.rs +++ /dev/null @@ -1,43 +0,0 @@ -use std::fmt::{Display, Formatter}; - -use serde::Deserialize; - -#[derive(Default, Debug, Clone, PartialEq, Eq, Hash)] -pub struct EmptyAuthCtx(); - -impl Display for EmptyAuthCtx { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "EmptyAuthCtx") - } -} - -impl IntoIterator for EmptyAuthCtx { - type Item = (String, String); - type IntoIter = std::iter::Empty; - - fn into_iter(self) -> Self::IntoIter { - std::iter::empty() - } -} - -#[derive( - Default, Debug, Clone, PartialEq, Eq, Hash, bincode::Encode, bincode::Decode, Deserialize, -)] -pub struct DefaultNamespace(); - -impl Display for DefaultNamespace { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "default") - } -} - -impl TryFrom for DefaultNamespace { - type Error = String; - fn try_from(value: String) -> Result { - if value.as_str() == "default" { - Ok(DefaultNamespace::default()) - } else { - Err("Failed to parse empty namespace".to_string()) - } - } -} diff --git a/golem-worker-service-base/src/lib.rs b/golem-worker-service-base/src/lib.rs index a2ed2369d9..4f4d333ec1 100644 --- a/golem-worker-service-base/src/lib.rs +++ b/golem-worker-service-base/src/lib.rs @@ -2,7 +2,6 @@ use ::http::Uri; pub mod api; pub mod api_definition; pub mod app_config; -pub mod auth; pub mod evaluator; pub mod http; mod merge; diff --git a/golem-worker-service-base/tests/services_tests.rs b/golem-worker-service-base/tests/services_tests.rs index cf4effea18..20f1001ce9 100644 --- a/golem-worker-service-base/tests/services_tests.rs +++ b/golem-worker-service-base/tests/services_tests.rs @@ -1,12 +1,12 @@ #[cfg(test)] mod tests { + use golem_service_base::auth::{DefaultNamespace, EmptyAuthCtx}; use golem_service_base::config::{DbPostgresConfig, DbSqliteConfig}; use golem_service_base::db; use golem_worker_service_base::api_definition::http::HttpApiDefinition; use golem_worker_service_base::api_definition::{ ApiDefinitionId, ApiDeployment, ApiSite, ApiSiteString, ApiVersion, }; - use golem_worker_service_base::auth::{DefaultNamespace, EmptyAuthCtx}; use golem_worker_service_base::repo::{api_definition, api_deployment}; use golem_worker_service_base::service::api_definition::{ ApiDefinitionError, ApiDefinitionIdWithVersion, ApiDefinitionService, diff --git a/golem-worker-service/src/api/api_definition.rs b/golem-worker-service/src/api/api_definition.rs index f1ada33edd..2df5257d34 100644 --- a/golem-worker-service/src/api/api_definition.rs +++ b/golem-worker-service/src/api/api_definition.rs @@ -8,12 +8,12 @@ use poem_openapi::*; use tracing::{error, info}; use golem_service_base::api_tags::ApiTags; +use golem_service_base::auth::{DefaultNamespace, EmptyAuthCtx}; use golem_worker_service_base::api::ApiEndpointError; use golem_worker_service_base::api::HttpApiDefinition; use golem_worker_service_base::api_definition::http::get_api_definition; use golem_worker_service_base::api_definition::http::HttpApiDefinition as CoreHttpApiDefinition; use golem_worker_service_base::api_definition::{ApiDefinitionId, ApiVersion}; -use golem_worker_service_base::auth::{DefaultNamespace, EmptyAuthCtx}; use golem_worker_service_base::service::api_definition::ApiDefinitionService; use golem_worker_service_base::service::http::http_api_definition_validator::RouteValidationError; diff --git a/golem-worker-service/src/api/api_deployment.rs b/golem-worker-service/src/api/api_deployment.rs index bd1182fae5..eaf526cf69 100644 --- a/golem-worker-service/src/api/api_deployment.rs +++ b/golem-worker-service/src/api/api_deployment.rs @@ -7,7 +7,7 @@ use poem_openapi::payload::Json; use poem_openapi::*; use golem_service_base::api_tags::ApiTags; -use golem_worker_service_base::auth::DefaultNamespace; +use golem_service_base::auth::DefaultNamespace; use tracing::log::info; use golem_worker_service_base::api::ApiDeployment; diff --git a/golem-worker-service/src/api/worker.rs b/golem-worker-service/src/api/worker.rs index d754ac836a..765f06362f 100644 --- a/golem-worker-service/src/api/worker.rs +++ b/golem-worker-service/src/api/worker.rs @@ -2,7 +2,7 @@ use golem_common::model::{ CallingConvention, ComponentId, IdempotencyKey, ScanCursor, WorkerFilter, }; use golem_service_base::api_tags::ApiTags; -use golem_worker_service_base::auth::EmptyAuthCtx; +use golem_service_base::auth::EmptyAuthCtx; use poem_openapi::param::{Header, Path, Query}; use poem_openapi::payload::Json; use poem_openapi::*; diff --git a/golem-worker-service/src/api/worker_connect.rs b/golem-worker-service/src/api/worker_connect.rs index d4bb0ac4c8..2413ff8ac0 100644 --- a/golem-worker-service/src/api/worker_connect.rs +++ b/golem-worker-service/src/api/worker_connect.rs @@ -16,8 +16,8 @@ use std::time::Duration; use futures::StreamExt; use golem_common::model::ComponentId; +use golem_service_base::auth::EmptyAuthCtx; use golem_service_base::model::WorkerId; -use golem_worker_service_base::auth::EmptyAuthCtx; use golem_worker_service_base::service::worker::{proxy_worker_connection, ConnectWorkerStream}; use poem::web::websocket::WebSocket; use poem::web::Data; diff --git a/golem-worker-service/src/grpcapi/api_definition.rs b/golem-worker-service/src/grpcapi/api_definition.rs index 4e56492b5c..c201df68e3 100644 --- a/golem-worker-service/src/grpcapi/api_definition.rs +++ b/golem-worker-service/src/grpcapi/api_definition.rs @@ -17,10 +17,9 @@ use golem_api_grpc::proto::golem::{ }, common::{Empty, ErrorBody, ErrorsBody}, }; -use golem_worker_service_base::auth::DefaultNamespace; +use golem_service_base::auth::{DefaultNamespace, EmptyAuthCtx}; use golem_worker_service_base::{ api_definition::{http::get_api_definition, ApiDefinitionId, ApiVersion}, - auth::EmptyAuthCtx, service::http::http_api_definition_validator::RouteValidationError, }; diff --git a/golem-worker-service/src/grpcapi/worker.rs b/golem-worker-service/src/grpcapi/worker.rs index 41f32053d0..cbad1697a7 100644 --- a/golem-worker-service/src/grpcapi/worker.rs +++ b/golem-worker-service/src/grpcapi/worker.rs @@ -31,7 +31,7 @@ use golem_api_grpc::proto::golem::worker::{ WorkerExecutionError, WorkerMetadata, }; use golem_common::model::{ComponentVersion, ScanCursor, WorkerFilter, WorkerId}; -use golem_worker_service_base::auth::EmptyAuthCtx; +use golem_service_base::auth::EmptyAuthCtx; use golem_worker_service_base::service::worker::ConnectWorkerStream; use tap::TapFallible; use tonic::{Request, Response, Status}; diff --git a/golem-worker-service/src/service/component.rs b/golem-worker-service/src/service/component.rs index 341ba6f890..7ad22af747 100644 --- a/golem-worker-service/src/service/component.rs +++ b/golem-worker-service/src/service/component.rs @@ -1,4 +1,4 @@ -use golem_worker_service_base::auth::EmptyAuthCtx; +use golem_service_base::auth::EmptyAuthCtx; use std::sync::Arc; pub type ComponentService = Arc< diff --git a/golem-worker-service/src/service/mod.rs b/golem-worker-service/src/service/mod.rs index 465f34ee87..d5240d8b31 100644 --- a/golem-worker-service/src/service/mod.rs +++ b/golem-worker-service/src/service/mod.rs @@ -5,8 +5,8 @@ use crate::worker_bridge_request_executor::UnauthorisedWorkerRequestExecutor; use golem_worker_service_base::api_definition::http::HttpApiDefinition; +use golem_service_base::auth::{DefaultNamespace, EmptyAuthCtx}; use golem_worker_service_base::app_config::WorkerServiceBaseConfig; -use golem_worker_service_base::auth::{DefaultNamespace, EmptyAuthCtx}; use golem_worker_service_base::http::InputHttpRequest; use golem_worker_service_base::repo::api_definition; diff --git a/golem-worker-service/src/service/worker.rs b/golem-worker-service/src/service/worker.rs index e7e0fd9ff6..fa5d1881d9 100644 --- a/golem-worker-service/src/service/worker.rs +++ b/golem-worker-service/src/service/worker.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use golem_worker_service_base::auth::EmptyAuthCtx; +use golem_service_base::auth::EmptyAuthCtx; pub type WorkerService = Arc + Sync + Send>; diff --git a/golem-worker-service/src/worker_bridge_request_executor.rs b/golem-worker-service/src/worker_bridge_request_executor.rs index 565ddc87ae..f6d96e423a 100644 --- a/golem-worker-service/src/worker_bridge_request_executor.rs +++ b/golem-worker-service/src/worker_bridge_request_executor.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use async_trait::async_trait; -use golem_worker_service_base::auth::EmptyAuthCtx; +use golem_service_base::auth::EmptyAuthCtx; use golem_worker_service_base::service::worker::WorkerService; use golem_worker_service_base::worker_bridge_execution::{ WorkerRequest, WorkerRequestExecutor, WorkerRequestExecutorError, WorkerResponse, @@ -32,9 +32,9 @@ mod internal { use crate::empty_worker_metadata; use crate::worker_bridge_request_executor::UnauthorisedWorkerRequestExecutor; use golem_common::model::CallingConvention; + use golem_service_base::auth::EmptyAuthCtx; use golem_service_base::model::WorkerId; use golem_wasm_rpc::json::get_json_from_typed_value; - use golem_worker_service_base::auth::EmptyAuthCtx; use serde_json::Value; use golem_worker_service_base::worker_bridge_execution::{ diff --git a/golem-worker-service/src/worker_component_metadata_fetcher.rs b/golem-worker-service/src/worker_component_metadata_fetcher.rs index aa235c0331..22a3c93c03 100644 --- a/golem-worker-service/src/worker_component_metadata_fetcher.rs +++ b/golem-worker-service/src/worker_component_metadata_fetcher.rs @@ -1,7 +1,7 @@ use crate::empty_worker_metadata; use async_trait::async_trait; +use golem_service_base::auth::EmptyAuthCtx; use golem_service_base::model::{ComponentMetadata, WorkerId}; -use golem_worker_service_base::auth::EmptyAuthCtx; use golem_worker_service_base::evaluator::{MetadataFetchError, WorkerMetadataFetcher}; use golem_worker_service_base::service::worker::WorkerService; use std::sync::Arc; From 95ff19f8b2fb59a927cbc9b6e8d28b1b1b53703d Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Fri, 28 Jun 2024 22:23:57 +0200 Subject: [PATCH 04/30] migrations scripts --- golem-component-service/db/migration/postgres/001__init.sql | 5 +++-- golem-component-service/db/migration/sqlite/001__init.sql | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/golem-component-service/db/migration/postgres/001__init.sql b/golem-component-service/db/migration/postgres/001__init.sql index 1a3f70fe8d..fe3714fac1 100644 --- a/golem-component-service/db/migration/postgres/001__init.sql +++ b/golem-component-service/db/migration/postgres/001__init.sql @@ -1,5 +1,6 @@ CREATE TABLE components ( + namespace text NOT NULL, component_id uuid NOT NULL, name text NOT NULL, size integer NOT NULL, @@ -8,6 +9,6 @@ CREATE TABLE components user_component text NOT NULL, protected_component text NOT NULL, protector_version bigint, - metadata jsonb NOT NULL, - PRIMARY KEY (component_id, version) + metadata bytea NOT NULL, + PRIMARY KEY (namespace, component_id, version) ); diff --git a/golem-component-service/db/migration/sqlite/001__init.sql b/golem-component-service/db/migration/sqlite/001__init.sql index 2648259cb3..673b3083df 100644 --- a/golem-component-service/db/migration/sqlite/001__init.sql +++ b/golem-component-service/db/migration/sqlite/001__init.sql @@ -1,5 +1,6 @@ CREATE TABLE components ( + namespace text NOT NULL, component_id uuid NOT NULL, name text NOT NULL, size integer NOT NULL, @@ -8,6 +9,6 @@ CREATE TABLE components user_component text NOT NULL, protected_component text NOT NULL, protector_version bigint, - metadata jsonb NOT NULL, - PRIMARY KEY (component_id, version) + metadata blob NOT NULL, + PRIMARY KEY (namespace, component_id, version) ); From a38c56f81b50e78cc5c166aff8fccf45aba915a4 Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Sat, 29 Jun 2024 20:41:22 +0200 Subject: [PATCH 05/30] test --- golem-component-service-base/Cargo.toml | 8 +- .../src/api/common.rs | 34 +++ golem-component-service-base/src/api/mod.rs | 1 + golem-component-service-base/src/lib.rs | 1 + .../tests/services_tests.rs | 205 ++++++++++++++++++ .../src/grpcapi/component.rs | 29 --- 6 files changed, 248 insertions(+), 30 deletions(-) create mode 100644 golem-component-service-base/src/api/common.rs create mode 100644 golem-component-service-base/src/api/mod.rs create mode 100644 golem-component-service-base/tests/services_tests.rs diff --git a/golem-component-service-base/Cargo.toml b/golem-component-service-base/Cargo.toml index f8b06e94ec..4ff276e192 100644 --- a/golem-component-service-base/Cargo.toml +++ b/golem-component-service-base/Cargo.toml @@ -32,4 +32,10 @@ tokio = { workspace = true } tokio-stream = { workspace = true } tokio-util = { workspace = true } tracing = { workspace = true } -uuid = { workspace = true } \ No newline at end of file +uuid = { workspace = true } + +[dev-dependencies] +criterion = { version = "0.3", features = ["html_reports"] } +fastrand = "2.0.2" +testcontainers = { workspace = true } +testcontainers-modules = { workspace = true } \ No newline at end of file diff --git a/golem-component-service-base/src/api/common.rs b/golem-component-service-base/src/api/common.rs new file mode 100644 index 0000000000..0e0daab7af --- /dev/null +++ b/golem-component-service-base/src/api/common.rs @@ -0,0 +1,34 @@ +mod conversion { + use crate::service::component; + use golem_api_grpc::proto::golem::common::{ErrorBody, ErrorsBody}; + use golem_api_grpc::proto::golem::component::{component_error, ComponentError}; + + impl From for ComponentError { + fn from(value: component::ComponentError) -> Self { + let error = match value { + component::ComponentError::AlreadyExists(_) => { + component_error::Error::AlreadyExists(ErrorBody { + error: value.to_string(), + }) + } + component::ComponentError::UnknownComponentId(_) + | component::ComponentError::UnknownVersionedComponentId(_) => { + component_error::Error::NotFound(ErrorBody { + error: value.to_string(), + }) + } + component::ComponentError::ComponentProcessingError(error) => { + component_error::Error::BadRequest(ErrorsBody { + errors: vec![error.to_string()], + }) + } + component::ComponentError::Internal(error) => { + component_error::Error::InternalError(ErrorBody { + error: error.to_string(), + }) + } + }; + ComponentError { error: Some(error) } + } + } +} diff --git a/golem-component-service-base/src/api/mod.rs b/golem-component-service-base/src/api/mod.rs new file mode 100644 index 0000000000..a5961aa6ba --- /dev/null +++ b/golem-component-service-base/src/api/mod.rs @@ -0,0 +1 @@ +mod common; diff --git a/golem-component-service-base/src/lib.rs b/golem-component-service-base/src/lib.rs index e369b480ee..3586d7f8a0 100644 --- a/golem-component-service-base/src/lib.rs +++ b/golem-component-service-base/src/lib.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub mod api; pub mod config; pub mod repo; pub mod service; diff --git a/golem-component-service-base/tests/services_tests.rs b/golem-component-service-base/tests/services_tests.rs new file mode 100644 index 0000000000..5fb1ee75c5 --- /dev/null +++ b/golem-component-service-base/tests/services_tests.rs @@ -0,0 +1,205 @@ +#[cfg(test)] +mod tests { + use golem_service_base::auth::DefaultNamespace; + use golem_service_base::config::{ComponentStoreLocalConfig, DbPostgresConfig, DbSqliteConfig}; + use golem_service_base::db; + + use golem_component_service_base::repo::component::{ComponentRepo, DbComponentRepo}; + use golem_component_service_base::service::component::{ + ComponentService, ComponentServiceDefault, + }; + use golem_component_service_base::service::component_compilation::{ + ComponentCompilationService, ComponentCompilationServiceDisabled, + }; + use golem_service_base::model::ComponentName; + use golem_service_base::service::component_object_store; + use std::sync::Arc; + use testcontainers::clients::Cli; + use testcontainers::{Container, RunnableImage}; + use testcontainers_modules::postgres::Postgres; + + fn start_docker_postgres<'d>(docker: &'d Cli) -> (DbPostgresConfig, Container<'d, Postgres>) { + let image = RunnableImage::from(Postgres::default()).with_tag("14.7-alpine"); + let container: Container<'d, Postgres> = docker.run(image); + + let config = DbPostgresConfig { + host: "localhost".to_string(), + port: container.get_host_port_ipv4(5432), + database: "postgres".to_string(), + username: "postgres".to_string(), + password: "postgres".to_string(), + schema: Some("test".to_string()), + max_connections: 10, + }; + + (config, container) + } + + struct SqliteDb { + db_path: String, + } + + impl Default for SqliteDb { + fn default() -> Self { + Self { + db_path: format!("/tmp/golem-component-{}.db", uuid::Uuid::new_v4()), + } + } + } + + impl Drop for SqliteDb { + fn drop(&mut self) { + std::fs::remove_file(&self.db_path).unwrap(); + } + } + + #[tokio::test] + pub async fn test_with_postgres_db() { + let cli = Cli::default(); + let (db_config, _container) = start_docker_postgres(&cli); + + db::postgres_migrate(&db_config, "tests/db/migration/postgres") + .await + .unwrap(); + + let db_pool = db::create_postgres_pool(&db_config).await.unwrap(); + + let component_repo: Arc = + Arc::new(DbComponentRepo::new(db_pool.clone().into())); + + test_services(component_repo).await; + } + + #[tokio::test] + pub async fn test_with_sqlite_db() { + let db = SqliteDb::default(); + let db_config = DbSqliteConfig { + database: db.db_path.clone(), + max_connections: 10, + }; + + db::sqlite_migrate(&db_config, "tests/db/migration/sqlite") + .await + .unwrap(); + + let db_pool = db::create_sqlite_pool(&db_config).await.unwrap(); + + let component_repo: Arc = + Arc::new(DbComponentRepo::new(db_pool.clone().into())); + + test_services(component_repo).await; + } + + async fn test_services(component_repo: Arc) { + let object_store: Arc = + Arc::new( + component_object_store::FsComponentObjectStore::new(&ComponentStoreLocalConfig { + root_path: "/tmp/component".to_string(), + object_prefix: uuid::Uuid::new_v4().to_string(), + }) + .unwrap(), + ); + + let compilation_service: Arc = + Arc::new(ComponentCompilationServiceDisabled); + + let component_service: Arc + Sync + Send> = + Arc::new(ComponentServiceDefault::new( + component_repo.clone(), + object_store.clone(), + compilation_service.clone(), + )); + + fn get_component_data(name: &str) -> Vec { + let path = format!("../test-components/{}.wasm", name); + std::fs::read(path).unwrap() + } + + let component_name1 = ComponentName("shopping-cart".to_string()); + let component_name2 = ComponentName("rust-echo".to_string()); + + let component1 = component_service + .create( + &component_name1, + get_component_data("shopping-cart"), + &DefaultNamespace::default(), + ) + .await + .unwrap(); + + let component2 = component_service + .create( + &component_name2, + get_component_data("rust-echo"), + &DefaultNamespace::default(), + ) + .await + .unwrap(); + + let component1_result = component_service + .get_by_version( + &component1.versioned_component_id, + &DefaultNamespace::default(), + ) + .await + .unwrap(); + assert!(component1_result.is_some()); + + let component2_result = component_service + .get_by_version( + &component2.versioned_component_id, + &DefaultNamespace::default(), + ) + .await + .unwrap(); + assert!(component2_result.is_some()); + assert_eq!(component2_result.unwrap(), component2); + + let component1_result = component_service + .get_latest_version( + &component1.versioned_component_id.component_id, + &DefaultNamespace::default(), + ) + .await + .unwrap(); + assert!(component1_result.is_some()); + assert_eq!(component1_result.unwrap(), component1); + + let component1_result = component_service + .get( + &component1.versioned_component_id.component_id, + &DefaultNamespace::default(), + ) + .await + .unwrap(); + assert_eq!(component1_result.len(), 1); + + let component1v2 = component_service + .update( + &component1.versioned_component_id.component_id, + get_component_data("shopping-cart"), + &DefaultNamespace::default(), + ) + .await + .unwrap(); + + let component1_result = component_service + .get_latest_version( + &component1.versioned_component_id.component_id, + &DefaultNamespace::default(), + ) + .await + .unwrap(); + assert!(component1_result.is_some()); + assert_eq!(component1_result.unwrap(), component1v2); + + let component1_result = component_service + .get( + &component1.versioned_component_id.component_id, + &DefaultNamespace::default(), + ) + .await + .unwrap(); + assert_eq!(component1_result.len(), 2); + } +} diff --git a/golem-component-service/src/grpcapi/component.rs b/golem-component-service/src/grpcapi/component.rs index 59a0de0de0..4ddfb2f1af 100644 --- a/golem-component-service/src/grpcapi/component.rs +++ b/golem-component-service/src/grpcapi/component.rs @@ -38,35 +38,6 @@ use tonic::{Request, Response, Status, Streaming}; use golem_component_service_base::service::component; use golem_service_base::auth::DefaultNamespace; -impl From for ComponentError { - fn from(value: component::ComponentError) -> Self { - let error = match value { - component::ComponentError::AlreadyExists(_) => { - component_error::Error::AlreadyExists(ErrorBody { - error: value.to_string(), - }) - } - component::ComponentError::UnknownComponentId(_) - | component::ComponentError::UnknownVersionedComponentId(_) => { - component_error::Error::NotFound(ErrorBody { - error: value.to_string(), - }) - } - component::ComponentError::ComponentProcessingError(error) => { - component_error::Error::BadRequest(ErrorsBody { - errors: vec![error.to_string()], - }) - } - component::ComponentError::Internal(error) => { - component_error::Error::InternalError(ErrorBody { - error: error.to_string(), - }) - } - }; - ComponentError { error: Some(error) } - } -} - fn bad_request_error(error: &str) -> ComponentError { ComponentError { error: Some(component_error::Error::BadRequest(ErrorsBody { From 538ae0f9638bf2435c9c53f4b12de14a9b645ea9 Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Sun, 30 Jun 2024 09:55:14 +0200 Subject: [PATCH 06/30] component-repo get all with namespace --- golem-component-service-base/src/repo/component.rs | 12 +++++++----- .../src/service/component.rs | 6 +++++- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/golem-component-service-base/src/repo/component.rs b/golem-component-service-base/src/repo/component.rs index b3719c152e..c42e16645c 100644 --- a/golem-component-service-base/src/repo/component.rs +++ b/golem-component-service-base/src/repo/component.rs @@ -102,7 +102,7 @@ pub trait ComponentRepo { component_id: &Uuid, ) -> Result, RepoError>; - async fn get_all(&self) -> Result, RepoError>; + async fn get_all(&self, namespace: &str) -> Result, RepoError>; async fn get_latest_version( &self, @@ -182,8 +182,9 @@ impl ComponentRepo for DbComponentRepo { .map_err(|e| e.into()) } - async fn get_all(&self) -> Result, RepoError> { - sqlx::query_as::<_, ComponentRecord>("SELECT namespace, component_id, version, name, size, user_component, protected_component, protector_version, metadata FROM components") + async fn get_all(&self, namespace: &str) -> Result, RepoError> { + sqlx::query_as::<_, ComponentRecord>("SELECT namespace, component_id, version, name, size, user_component, protected_component, protector_version, metadata FROM components WHERE namespace = $1") + .bind(namespace) .fetch_all(self.db_pool.deref()) .await .map_err(|e| e.into()) @@ -278,8 +279,9 @@ impl ComponentRepo for DbComponentRepo { Ok(()) } - async fn get_all(&self) -> Result, RepoError> { - sqlx::query_as::<_, ComponentRecord>("SELECT namespace, component_id, name, size, version, user_component, protected_component, protector_version, metadata FROM components") + async fn get_all(&self, namespace: &str) -> Result, RepoError> { + sqlx::query_as::<_, ComponentRecord>("SELECT namespace, component_id, name, size, version, user_component, protected_component, protector_version, metadata FROM components WHERE namespace = $1") + .bind(namespace) .fetch_all(self.db_pool.deref()) .await .map_err(|e| e.into()) diff --git a/golem-component-service-base/src/service/component.rs b/golem-component-service-base/src/service/component.rs index 5b3b167c1b..c27bc2cea4 100644 --- a/golem-component-service-base/src/service/component.rs +++ b/golem-component-service-base/src/service/component.rs @@ -398,7 +398,11 @@ impl + Eq + Clone + Send + Sync> ComponentS .get_by_name(namespace.to_string().as_str(), &name.0) .await? } - None => self.component_repo.get_all().await?, + None => { + self.component_repo + .get_all(namespace.to_string().as_str()) + .await? + } }; let values: Vec = records From ea834558f53f68019165ef5a07c562d6f471b5f3 Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Sun, 30 Jun 2024 10:18:22 +0200 Subject: [PATCH 07/30] component-repo get_namespaces --- .../src/repo/component.rs | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/golem-component-service-base/src/repo/component.rs b/golem-component-service-base/src/repo/component.rs index c42e16645c..e41c3b1e4b 100644 --- a/golem-component-service-base/src/repo/component.rs +++ b/golem-component-service-base/src/repo/component.rs @@ -123,6 +123,8 @@ pub trait ComponentRepo { name: &str, ) -> Result, RepoError>; + async fn get_namespaces(&self, component_id: &Uuid) -> Result, RepoError>; + async fn delete(&self, namespace: &str, component_id: &Uuid) -> Result<(), RepoError>; } @@ -236,6 +238,16 @@ impl ComponentRepo for DbComponentRepo { .map_err(|e| e.into()) } + async fn get_namespaces(&self, component_id: &Uuid) -> Result, RepoError> { + sqlx::query_as::<_, (String, i64)>( + "SELECT namespace, max(version) FROM components WHERE component_id = $1 GROUP BY namespace", + ) + .bind(component_id) + .fetch_all(self.db_pool.deref()) + .await + .map_err(|e| e.into()) + } + async fn delete(&self, namespace: &str, component_id: &Uuid) -> Result<(), RepoError> { sqlx::query("DELETE FROM components WHERE namespace = $1 AND component_id = $2") .bind(namespace) @@ -347,6 +359,16 @@ impl ComponentRepo for DbComponentRepo { .map_err(|e| e.into()) } + async fn get_namespaces(&self, component_id: &Uuid) -> Result, RepoError> { + sqlx::query_as::<_, (String, i64)>( + "SELECT namespace, max(version) FROM components WHERE component_id = $1 GROUP BY namespace", + ) + .bind(component_id) + .fetch_all(self.db_pool.deref()) + .await + .map_err(|e| e.into()) + } + async fn delete(&self, namespace: &str, component_id: &Uuid) -> Result<(), RepoError> { sqlx::query("DELETE FROM components WHERE namespace = $1 AND component_id = $2") .bind(namespace) From 8ef1638df2a58815a4e88a4ce9f6636c267648e0 Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Sun, 30 Jun 2024 17:16:13 +0200 Subject: [PATCH 08/30] component-service get_namespace --- .../src/service/component.rs | 42 +++++++++++++++++-- .../tests/services_tests.rs | 14 +++++++ golem-service-base/src/model.rs | 4 -- 3 files changed, 53 insertions(+), 7 deletions(-) diff --git a/golem-component-service-base/src/service/component.rs b/golem-component-service-base/src/service/component.rs index c27bc2cea4..4d423cad51 100644 --- a/golem-component-service-base/src/service/component.rs +++ b/golem-component-service-base/src/service/component.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::fmt::Display; +use std::fmt::{Debug, Display}; use std::sync::Arc; use crate::service::component_compilation::ComponentCompilationService; @@ -118,6 +118,11 @@ pub trait ComponentService { component_id: &ComponentId, namespace: &Namespace, ) -> Result, ComponentError>; + + async fn get_namespace( + &self, + component_id: &ComponentId, + ) -> Result, ComponentError>; } pub struct ComponentServiceDefault { @@ -141,8 +146,10 @@ impl ComponentServiceDefault { } #[async_trait] -impl + Eq + Clone + Send + Sync> ComponentService - for ComponentServiceDefault +impl ComponentService for ComponentServiceDefault +where + Namespace: Display + TryFrom + Eq + Clone + Send + Sync, + >::Error: Display + Debug + Send + Sync + 'static, { async fn create( &self, @@ -485,6 +492,28 @@ impl + Eq + Clone + Send + Sync> ComponentS None => Ok(None), } } + + async fn get_namespace( + &self, + component_id: &ComponentId, + ) -> Result, ComponentError> { + info!("Getting component {} namespace", component_id); + let result = self.component_repo.get_namespaces(&component_id.0).await?; + + if result.is_empty() { + Ok(None) + } else if result.len() == 1 { + let value = result[0].clone().0.try_into().map_err(|e| { + ComponentError::internal(e, "Failed to convert namespace".to_string()) + })?; + Ok(Some(value)) + } else { + Err(ComponentError::internal( + "", + "Namespace is not unique".to_string(), + )) + } + } } impl ComponentServiceDefault { @@ -682,4 +711,11 @@ impl + Eq + Clone + Send + Sync> ComponentS ) -> Result, ComponentError> { Ok(vec![]) } + + async fn get_namespace( + &self, + _component_id: &ComponentId, + ) -> Result, ComponentError> { + Ok(None) + } } diff --git a/golem-component-service-base/tests/services_tests.rs b/golem-component-service-base/tests/services_tests.rs index 5fb1ee75c5..ba1359b014 100644 --- a/golem-component-service-base/tests/services_tests.rs +++ b/golem-component-service-base/tests/services_tests.rs @@ -201,5 +201,19 @@ mod tests { .await .unwrap(); assert_eq!(component1_result.len(), 2); + + let component1_result = component_service + .get_namespace(&component1.versioned_component_id.component_id) + .await + .unwrap(); + assert!(component1_result.is_some()); + assert_eq!(component1_result.unwrap(), DefaultNamespace::default()); + + let component2_result = component_service + .get_namespace(&component1.versioned_component_id.component_id) + .await + .unwrap(); + assert!(component2_result.is_some()); + assert_eq!(component2_result.unwrap(), DefaultNamespace::default()); } } diff --git a/golem-service-base/src/model.rs b/golem-service-base/src/model.rs index 544ced33d5..526992d8bf 100644 --- a/golem-service-base/src/model.rs +++ b/golem-service-base/src/model.rs @@ -1172,8 +1172,6 @@ impl From for golem_wasm_rpc::protobuf::Type { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Object, Encode, Decode)] pub struct FunctionParameter { pub name: String, - // TODO: Fix this in DB. Temp fix for now. - #[serde(rename = "tpe")] pub typ: Type, } @@ -1202,8 +1200,6 @@ impl From for golem_api_grpc::proto::golem::component::Functi #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Object, Encode, Decode)] pub struct FunctionResult { pub name: Option, - // TODO: Fix this in DB. Temp fix for now. - #[serde(rename = "tpe")] pub typ: Type, } From 5b7b0e120b7ece1a215740fb044cbbe538d4dd65 Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Sun, 30 Jun 2024 20:20:46 +0200 Subject: [PATCH 09/30] component-service get_ids_by_name --- .../src/repo/component.rs | 28 ++++++++++- .../src/service/component.rs | 50 ++++++++++++------- 2 files changed, 59 insertions(+), 19 deletions(-) diff --git a/golem-component-service-base/src/repo/component.rs b/golem-component-service-base/src/repo/component.rs index e41c3b1e4b..6f0235dbc6 100644 --- a/golem-component-service-base/src/repo/component.rs +++ b/golem-component-service-base/src/repo/component.rs @@ -19,7 +19,7 @@ use std::sync::Arc; use async_trait::async_trait; use golem_common::model::ComponentId; -use sqlx::{Database, Pool}; +use sqlx::{Database, Pool, Row}; use uuid::Uuid; use crate::repo::RepoError; @@ -123,6 +123,8 @@ pub trait ComponentRepo { name: &str, ) -> Result, RepoError>; + async fn get_ids_by_name(&self, namespace: &str, name: &str) -> Result, RepoError>; + async fn get_namespaces(&self, component_id: &Uuid) -> Result, RepoError>; async fn delete(&self, namespace: &str, component_id: &Uuid) -> Result<(), RepoError>; @@ -238,6 +240,18 @@ impl ComponentRepo for DbComponentRepo { .map_err(|e| e.into()) } + async fn get_ids_by_name(&self, namespace: &str, name: &str) -> Result, RepoError> { + let result = sqlx::query( + "SELECT distinct component_id as component_id FROM components WHERE namespace = $1 AND name = $2" + ) + .bind(namespace) + .bind(name) + .fetch_all(self.db_pool.deref()) + .await?; + + Ok(result.into_iter().map(|x| x.get("component_id")).collect()) + } + async fn get_namespaces(&self, component_id: &Uuid) -> Result, RepoError> { sqlx::query_as::<_, (String, i64)>( "SELECT namespace, max(version) FROM components WHERE component_id = $1 GROUP BY namespace", @@ -327,6 +341,18 @@ impl ComponentRepo for DbComponentRepo { .map_err(|e| e.into()) } + async fn get_ids_by_name(&self, namespace: &str, name: &str) -> Result, RepoError> { + let result = sqlx::query( + "SELECT distinct component_id as component_id FROM components WHERE namespace = $1 AND name = $2" + ) + .bind(namespace) + .bind(name) + .fetch_all(self.db_pool.deref()) + .await?; + + Ok(result.into_iter().map(|x| x.get("component_id")).collect()) + } + async fn get_latest_version( &self, namespace: &str, diff --git a/golem-component-service-base/src/service/component.rs b/golem-component-service-base/src/service/component.rs index 4d423cad51..3116b21704 100644 --- a/golem-component-service-base/src/service/component.rs +++ b/golem-component-service-base/src/service/component.rs @@ -101,6 +101,12 @@ pub trait ComponentService { namespace: &Namespace, ) -> Result, ComponentError>; + async fn find_ids_by_name( + &self, + component_name: &ComponentName, + namespace: &Namespace, + ) -> Result, ComponentError>; + async fn get_by_version( &self, component_id: &VersionedComponentId, @@ -110,7 +116,7 @@ pub trait ComponentService { async fn get_latest_version( &self, component_id: &ComponentId, - _namespace: &Namespace, + namespace: &Namespace, ) -> Result, ComponentError>; async fn get( @@ -160,7 +166,11 @@ where let tn = component_name.0.clone(); info!("Creating component with name {}", tn); - self.check_new_name(component_name, namespace).await?; + self.find_ids_by_name(component_name, namespace) + .await? + .into_iter() + .next() + .map_or(Ok(()), |id| Err(ComponentError::AlreadyExists(id)))?; let metadata = process_component(&data)?; @@ -391,6 +401,18 @@ where Ok(Some(result)) } + async fn find_ids_by_name( + &self, + component_name: &ComponentName, + namespace: &Namespace, + ) -> Result, ComponentError> { + let records = self + .component_repo + .get_ids_by_name(namespace.to_string().as_str(), &component_name.0) + .await?; + Ok(records.into_iter().map(ComponentId).collect()) + } + async fn find_by_name( &self, component_name: Option, @@ -517,22 +539,6 @@ where } impl ComponentServiceDefault { - async fn check_new_name( - &self, - component_name: &ComponentName, - namespace: &Namespace, - ) -> Result<(), ComponentError> { - let existing_components = self - .component_repo - .get_by_name(namespace.to_string().as_str(), &component_name.0) - .await - .tap_err(|e| error!("Error getting existing components: {}", e))?; - - existing_components.into_iter().next().map_or(Ok(()), |c| { - Err(ComponentError::AlreadyExists(ComponentId(c.component_id))) - }) - } - fn get_user_object_store_key(&self, id: &UserComponentId) -> String { id.slug() } @@ -671,6 +677,14 @@ impl + Eq + Clone + Send + Sync> ComponentS Ok(ByteStream::empty()) } + async fn find_ids_by_name( + &self, + _component_name: &ComponentName, + _namespace: &Namespace, + ) -> Result, ComponentError> { + Ok(vec![]) + } + async fn get_protected_data( &self, _component_id: &ComponentId, From 71a917e91b9ed2bacb9916ab793d19eaa5d1364f Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Mon, 1 Jul 2024 18:44:11 +0200 Subject: [PATCH 10/30] component-service logs, namespace check --- .../src/repo/component.rs | 42 +++++++++++++- .../src/service/component.rs | 57 ++++++++++++------- 2 files changed, 77 insertions(+), 22 deletions(-) diff --git a/golem-component-service-base/src/repo/component.rs b/golem-component-service-base/src/repo/component.rs index 6f0235dbc6..f09e7abf12 100644 --- a/golem-component-service-base/src/repo/component.rs +++ b/golem-component-service-base/src/repo/component.rs @@ -143,6 +143,22 @@ impl DbComponentRepo { #[async_trait] impl ComponentRepo for DbComponentRepo { async fn upsert(&self, component: &ComponentRecord) -> Result<(), RepoError> { + let mut transaction = self.db_pool.begin().await?; + + let result = sqlx::query( + "SELECT count(component_id) as count FROM components WHERE namespace != $1 AND component_id = $2", + ) + .bind(component.namespace.clone()) + .bind(component.component_id) + .fetch_one(&mut *transaction) + .await?; + + let count: i64 = result.get("count"); + + if count > 0 { + return Err(RepoError::Internal("Component id not unique".to_string())); + } + sqlx::query( r#" INSERT INTO components @@ -167,9 +183,11 @@ impl ComponentRepo for DbComponentRepo { .bind(component.protected_component.clone()) .bind(component.protector_version) .bind(component.metadata.clone()) - .execute(self.db_pool.deref()) + .execute(&mut *transaction) .await?; + transaction.commit().await?; + Ok(()) } @@ -275,6 +293,22 @@ impl ComponentRepo for DbComponentRepo { #[async_trait] impl ComponentRepo for DbComponentRepo { async fn upsert(&self, component: &ComponentRecord) -> Result<(), RepoError> { + let mut transaction = self.db_pool.begin().await?; + + let result = sqlx::query( + "SELECT count(component_id) as count FROM components WHERE namespace != $1 AND component_id = $2", + ) + .bind(component.namespace.clone()) + .bind(component.component_id) + .fetch_one(&mut *transaction) + .await?; + + let count: i64 = result.get("count"); + + if count > 0 { + return Err(RepoError::Internal("Component id not unique".to_string())); + } + sqlx::query( r#" INSERT INTO components @@ -288,7 +322,7 @@ impl ComponentRepo for DbComponentRepo { protected_component = $7, protector_version = $8, metadata = $9 - "#, + "#, ) .bind(component.namespace.clone()) .bind(component.component_id) @@ -299,9 +333,11 @@ impl ComponentRepo for DbComponentRepo { .bind(component.protected_component.clone()) .bind(component.protector_version) .bind(component.metadata.clone()) - .execute(self.db_pool.deref()) + .execute(&mut *transaction) .await?; + transaction.commit().await?; + Ok(()) } diff --git a/golem-component-service-base/src/service/component.rs b/golem-component-service-base/src/service/component.rs index 3116b21704..0570ecef6b 100644 --- a/golem-component-service-base/src/service/component.rs +++ b/golem-component-service-base/src/service/component.rs @@ -163,8 +163,11 @@ where data: Vec, namespace: &Namespace, ) -> Result { - let tn = component_name.0.clone(); - info!("Creating component with name {}", tn); + info!( + "Creating component - namespace: {}, name: {}", + namespace, + component_name.0.clone() + ); self.find_ids_by_name(component_name, namespace) .await? @@ -189,8 +192,8 @@ where }; info!( - "Uploaded component {} version 0 with exports {:?}", - versioned_component_id.component_id, metadata.exports + "Uploaded component - namespace: {}, id: {}, version: 0, exports {:?}", + namespace, versioned_component_id.component_id, metadata.exports ); let component_size: u64 = data @@ -230,7 +233,10 @@ where data: Vec, namespace: &Namespace, ) -> Result { - info!("Updating component {}", component_id); + info!( + "Updating component - namespace: {}, id: {}", + namespace, component_id + ); let metadata = process_component(&data)?; @@ -246,8 +252,11 @@ where .map(Component::next_version)?; info!( - "Uploaded component {} version {} with exports {:?}", - component_id, next_component.versioned_component_id.version, metadata.exports + "Uploaded component - namespace: {}, id: {}, version: {}, exports {:?}", + namespace, + component_id, + next_component.versioned_component_id.version, + metadata.exports ); let component_size: u64 = data @@ -298,8 +307,8 @@ where } }; info!( - "Downloading component {} version {}", - component_id, versioned_component_id.version + "Downloading component - namespace: {}, id: {}, version: {}", + namespace, component_id, versioned_component_id.version ); let id = ProtectedComponentId { @@ -334,8 +343,8 @@ where } }; info!( - "Downloading component {} version {}", - component_id, versioned_component_id.version + "Downloading component - namespace: {}, id: {}, version: {}", + namespace, component_id, versioned_component_id.version ); let id = ProtectedComponentId { @@ -357,7 +366,8 @@ where namespace: &Namespace, ) -> Result>, ComponentError> { info!( - "Getting component {} version {} data", + "Getting component data - namespace: {}, id: {}, version: {}", + namespace, component_id, version.map_or("N/A".to_string(), |v| v.to_string()) ); @@ -418,8 +428,11 @@ where component_name: Option, namespace: &Namespace, ) -> Result, ComponentError> { - let tn = component_name.clone().map_or("N/A".to_string(), |n| n.0); - info!("Getting component name {}", tn); + let cn = component_name.clone().map_or("N/A".to_string(), |n| n.0); + info!( + "Find component by name - namespace: {}, name: {}", + namespace, cn + ); let records = match component_name { Some(name) => { @@ -448,7 +461,10 @@ where component_id: &ComponentId, namespace: &Namespace, ) -> Result, ComponentError> { - info!("Getting component {}", component_id); + info!( + "Getting component - namespace: {}, id: {}", + namespace, component_id + ); let records = self .component_repo .get(namespace.to_string().as_str(), &component_id.0) @@ -469,8 +485,8 @@ where namespace: &Namespace, ) -> Result, ComponentError> { info!( - "Getting component {} version {}", - component_id.component_id, component_id.version + "Getting component - namespace: {}, id: {}, version: {}", + namespace, component_id.component_id, component_id.version ); let result = self @@ -498,7 +514,10 @@ where component_id: &ComponentId, namespace: &Namespace, ) -> Result, ComponentError> { - info!("Getting component {} latest version", component_id); + info!( + "Getting component - namespace: {}, id: {}, version: latest", + namespace, component_id + ); let result = self .component_repo .get_latest_version(namespace.to_string().as_str(), &component_id.0) @@ -519,7 +538,7 @@ where &self, component_id: &ComponentId, ) -> Result, ComponentError> { - info!("Getting component {} namespace", component_id); + info!("Getting component namespace - id: {}", component_id); let result = self.component_repo.get_namespaces(&component_id.0).await?; if result.is_empty() { From 1452b2743a05377966e0fe67c404ff7cf9519505 Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Mon, 1 Jul 2024 19:07:23 +0200 Subject: [PATCH 11/30] component-service create with id --- .../src/service/component.rs | 37 +++++++++++-------- .../tests/services_tests.rs | 3 ++ golem-component-service/src/api/component.rs | 7 +++- .../src/grpcapi/component.rs | 7 +++- 4 files changed, 36 insertions(+), 18 deletions(-) diff --git a/golem-component-service-base/src/service/component.rs b/golem-component-service-base/src/service/component.rs index 0570ecef6b..e914f2d43b 100644 --- a/golem-component-service-base/src/service/component.rs +++ b/golem-component-service-base/src/service/component.rs @@ -62,6 +62,7 @@ impl From for ComponentError { pub trait ComponentService { async fn create( &self, + component_id: &ComponentId, component_name: &ComponentName, data: Vec, namespace: &Namespace, @@ -159,13 +160,15 @@ where { async fn create( &self, + component_id: &ComponentId, component_name: &ComponentName, data: Vec, namespace: &Namespace, ) -> Result { info!( - "Creating component - namespace: {}, name: {}", + "Creating component - namespace: {}, id: {}, name: {}", namespace, + component_id, component_name.0.clone() ); @@ -177,10 +180,8 @@ where let metadata = process_component(&data)?; - let component_id = ComponentId::new_v4(); - let versioned_component_id = VersionedComponentId { - component_id, + component_id: component_id.clone(), version: 0, }; @@ -571,7 +572,10 @@ impl ComponentServiceDefault { user_component_id: &UserComponentId, data: Vec, ) -> Result<(), ComponentError> { - info!("Uploading user component: {:?}", user_component_id); + info!( + "Uploading user component - id: {}", + user_component_id.slug() + ); self.object_store .put(&self.get_user_object_store_key(user_component_id), data) @@ -585,8 +589,8 @@ impl ComponentServiceDefault { data: Vec, ) -> Result<(), ComponentError> { info!( - "Uploading protected component: {:?}", - protected_component_id + "Uploading protected component - id: {}", + protected_component_id.slug() ); self.object_store @@ -610,12 +614,13 @@ impl + Eq + Clone + Send + Sync> ComponentS { async fn create( &self, - _component_name: &ComponentName, + component_id: &ComponentId, + component_name: &ComponentName, _data: Vec, _namespace: &Namespace, ) -> Result { let fake_component = Component { - component_name: ComponentName("fake".to_string()), + component_name: component_name.clone(), component_size: 0, metadata: ComponentMetadata { exports: vec![], @@ -623,18 +628,18 @@ impl + Eq + Clone + Send + Sync> ComponentS memories: vec![], }, versioned_component_id: VersionedComponentId { - component_id: ComponentId::new_v4(), + component_id: component_id.clone(), version: 0, }, user_component_id: UserComponentId { versioned_component_id: VersionedComponentId { - component_id: ComponentId::new_v4(), + component_id: component_id.clone(), version: 0, }, }, protected_component_id: ProtectedComponentId { versioned_component_id: VersionedComponentId { - component_id: ComponentId::new_v4(), + component_id: component_id.clone(), version: 0, }, }, @@ -645,7 +650,7 @@ impl + Eq + Clone + Send + Sync> ComponentS async fn update( &self, - _component_id: &ComponentId, + component_id: &ComponentId, _data: Vec, _namespace: &Namespace, ) -> Result { @@ -658,18 +663,18 @@ impl + Eq + Clone + Send + Sync> ComponentS memories: vec![], }, versioned_component_id: VersionedComponentId { - component_id: ComponentId::new_v4(), + component_id: component_id.clone(), version: 0, }, user_component_id: UserComponentId { versioned_component_id: VersionedComponentId { - component_id: ComponentId::new_v4(), + component_id: component_id.clone(), version: 0, }, }, protected_component_id: ProtectedComponentId { versioned_component_id: VersionedComponentId { - component_id: ComponentId::new_v4(), + component_id: component_id.clone(), version: 0, }, }, diff --git a/golem-component-service-base/tests/services_tests.rs b/golem-component-service-base/tests/services_tests.rs index ba1359b014..3839a1bfec 100644 --- a/golem-component-service-base/tests/services_tests.rs +++ b/golem-component-service-base/tests/services_tests.rs @@ -4,6 +4,7 @@ mod tests { use golem_service_base::config::{ComponentStoreLocalConfig, DbPostgresConfig, DbSqliteConfig}; use golem_service_base::db; + use golem_common::model::ComponentId; use golem_component_service_base::repo::component::{ComponentRepo, DbComponentRepo}; use golem_component_service_base::service::component::{ ComponentService, ComponentServiceDefault, @@ -120,6 +121,7 @@ mod tests { let component1 = component_service .create( + &ComponentId::new_v4(), &component_name1, get_component_data("shopping-cart"), &DefaultNamespace::default(), @@ -129,6 +131,7 @@ mod tests { let component2 = component_service .create( + &ComponentId::new_v4(), &component_name2, get_component_data("rust-echo"), &DefaultNamespace::default(), diff --git a/golem-component-service/src/api/component.rs b/golem-component-service/src/api/component.rs index 55683a0dc6..3d5b68be5c 100644 --- a/golem-component-service/src/api/component.rs +++ b/golem-component-service/src/api/component.rs @@ -109,7 +109,12 @@ impl ComponentApi { let component_name = payload.name; let response = self .component_service - .create(&component_name, data, &DefaultNamespace::default()) + .create( + &ComponentId::new_v4(), + &component_name, + data, + &DefaultNamespace::default(), + ) .await?; Ok(Json(response)) } diff --git a/golem-component-service/src/grpcapi/component.rs b/golem-component-service/src/grpcapi/component.rs index 4ddfb2f1af..beef19c650 100644 --- a/golem-component-service/src/grpcapi/component.rs +++ b/golem-component-service/src/grpcapi/component.rs @@ -154,7 +154,12 @@ impl ComponentGrpcApi { let name = golem_service_base::model::ComponentName(request.component_name); let result = self .component_service - .create(&name, data, &DefaultNamespace::default()) + .create( + &ComponentId::new_v4(), + &name, + data, + &DefaultNamespace::default(), + ) .await?; Ok(result.into()) } From ddc25e99f96bcac02f81e6f73b4cbecf90d82d1f Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Tue, 2 Jul 2024 09:44:06 +0200 Subject: [PATCH 12/30] naming fix - tpe -> typ --- .../proto/golem/component/function_parameter.proto | 2 +- .../proto/golem/component/function_result.proto | 2 +- golem-service-base/src/model.rs | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/golem-api-grpc/proto/golem/component/function_parameter.proto b/golem-api-grpc/proto/golem/component/function_parameter.proto index ba16eb88da..2bf72f8652 100644 --- a/golem-api-grpc/proto/golem/component/function_parameter.proto +++ b/golem-api-grpc/proto/golem/component/function_parameter.proto @@ -6,5 +6,5 @@ import "wasm/rpc/type.proto"; message FunctionParameter { string name = 1; - wasm.rpc.Type tpe = 2; + wasm.rpc.Type typ = 2; } diff --git a/golem-api-grpc/proto/golem/component/function_result.proto b/golem-api-grpc/proto/golem/component/function_result.proto index f3de933539..fb3f90f450 100644 --- a/golem-api-grpc/proto/golem/component/function_result.proto +++ b/golem-api-grpc/proto/golem/component/function_result.proto @@ -6,5 +6,5 @@ import "wasm/rpc/type.proto"; message FunctionResult { optional string name = 1; - wasm.rpc.Type tpe = 2; + wasm.rpc.Type typ = 2; } diff --git a/golem-service-base/src/model.rs b/golem-service-base/src/model.rs index 526992d8bf..8a3c108927 100644 --- a/golem-service-base/src/model.rs +++ b/golem-service-base/src/model.rs @@ -1183,7 +1183,7 @@ impl TryFrom for Fun ) -> Result { Ok(Self { name: value.name, - typ: value.tpe.ok_or("Missing tpe")?.try_into()?, + typ: value.typ.ok_or("Missing typ")?.try_into()?, }) } } @@ -1192,7 +1192,7 @@ impl From for golem_api_grpc::proto::golem::component::Functi fn from(value: FunctionParameter) -> Self { Self { name: value.name, - tpe: Some(value.typ.into()), + typ: Some(value.typ.into()), } } } @@ -1211,7 +1211,7 @@ impl TryFrom for Functi ) -> Result { Ok(Self { name: value.name, - typ: value.tpe.ok_or("Missing tpe")?.try_into()?, + typ: value.typ.ok_or("Missing typ")?.try_into()?, }) } } @@ -1220,7 +1220,7 @@ impl From for golem_api_grpc::proto::golem::component::FunctionR fn from(value: FunctionResult) -> Self { Self { name: value.name, - tpe: Some(value.typ.into()), + typ: Some(value.typ.into()), } } } From 2a8dacf282137d331c8be9f8b38b34ef1ec80512 Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Tue, 2 Jul 2024 18:19:18 +0200 Subject: [PATCH 13/30] components and component_versions tables, repo update --- .../src/repo/component.rs | 191 +++++++++++------- .../src/service/component.rs | 2 +- .../tests/db/migration/postgres/001__init.sql | 15 +- .../tests/db/migration/sqlite/001__init.sql | 16 +- .../tests/services_tests.rs | 32 ++- .../db/migration/postgres/001__init.sql | 15 +- .../db/migration/sqlite/001__init.sql | 16 +- 7 files changed, 199 insertions(+), 88 deletions(-) diff --git a/golem-component-service-base/src/repo/component.rs b/golem-component-service-base/src/repo/component.rs index f09e7abf12..ff55ae0036 100644 --- a/golem-component-service-base/src/repo/component.rs +++ b/golem-component-service-base/src/repo/component.rs @@ -125,7 +125,7 @@ pub trait ComponentRepo { async fn get_ids_by_name(&self, namespace: &str, name: &str) -> Result, RepoError>; - async fn get_namespaces(&self, component_id: &Uuid) -> Result, RepoError>; + async fn get_namespaces(&self, component_id: &Uuid) -> Result, RepoError>; async fn delete(&self, namespace: &str, component_id: &Uuid) -> Result<(), RepoError>; } @@ -162,22 +162,34 @@ impl ComponentRepo for DbComponentRepo { sqlx::query( r#" INSERT INTO components - (namespace, component_id, version, name, size, user_component, protected_component, protector_version, metadata) + (namespace, component_id, name) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9) - ON CONFLICT (namespace, component_id, version) DO UPDATE - SET name = $4, - size = $5, - user_component = $6, - protected_component = $7, - protector_version = $8, - metadata = $9 + ($1, $2, $3) + ON CONFLICT (namespace, name) DO NOTHING + "#, + ) + .bind(component.namespace.clone()) + .bind(component.component_id) + .bind(component.name.clone()) + .execute(&mut *transaction) + .await?; + + sqlx::query( + r#" + INSERT INTO component_versions + (component_id, version, size, user_component, protected_component, protector_version, metadata) + VALUES + ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (component_id, version) DO UPDATE + SET size = $3, + user_component = $4, + protected_component = $5, + protector_version = $6, + metadata = $7 "#, ) - .bind(component.namespace.clone()) .bind(component.component_id) .bind(component.version) - .bind(component.name.clone()) .bind(component.size) .bind(component.user_component.clone()) .bind(component.protected_component.clone()) @@ -196,7 +208,9 @@ impl ComponentRepo for DbComponentRepo { namespace: &str, component_id: &Uuid, ) -> Result, RepoError> { - sqlx::query_as::<_, ComponentRecord>("SELECT namespace, component_id, version, name, size, user_component, protected_component, protector_version, metadata FROM components WHERE namespace = $1 AND component_id = $2") + sqlx::query_as::<_, ComponentRecord>( + "SELECT components.namespace, components.name, component_versions.* FROM components JOIN component_versions ON components.component_id = component_versions.component_id WHERE components.namespace = $1 AND component_versions.component_id = $2" + ) .bind(namespace) .bind(component_id) .fetch_all(self.db_pool.deref()) @@ -205,7 +219,7 @@ impl ComponentRepo for DbComponentRepo { } async fn get_all(&self, namespace: &str) -> Result, RepoError> { - sqlx::query_as::<_, ComponentRecord>("SELECT namespace, component_id, version, name, size, user_component, protected_component, protector_version, metadata FROM components WHERE namespace = $1") + sqlx::query_as::<_, ComponentRecord>("SELECT components.namespace, components.name, component_versions.* FROM components JOIN component_versions ON components.component_id = component_versions.component_id WHERE components.namespace = $1") .bind(namespace) .fetch_all(self.db_pool.deref()) .await @@ -218,7 +232,7 @@ impl ComponentRepo for DbComponentRepo { component_id: &Uuid, ) -> Result, RepoError> { sqlx::query_as::<_, ComponentRecord>( - "SELECT namespace, component_id, version, name, size, user_component, protected_component, protector_version, metadata FROM components WHERE namespace = $1 AND component_id = $2 ORDER BY version DESC LIMIT 1", + "SELECT components.namespace, components.name, component_versions.* FROM components JOIN component_versions ON components.component_id = component_versions.component_id WHERE components.namespace = $1 AND component_versions.component_id = $2 ORDER BY version DESC LIMIT 1", ).bind(namespace) .bind(component_id) .fetch_optional(self.db_pool.deref()) @@ -233,7 +247,7 @@ impl ComponentRepo for DbComponentRepo { version: u64, ) -> Result, RepoError> { sqlx::query_as::<_, ComponentRecord>( - "SELECT namespace, component_id, version, name, size, user_component, protected_component, protector_version, metadata FROM components WHERE namespace = $1 AND component_id = $2 AND version = $3", + "SELECT components.namespace, components.name, component_versions.* FROM components JOIN component_versions ON components.component_id = component_versions.component_id WHERE components.namespace = $1 AND component_versions.component_id = $2 AND component_versions.version = $3", ) .bind(namespace) .bind(component_id) @@ -249,7 +263,7 @@ impl ComponentRepo for DbComponentRepo { name: &str, ) -> Result, RepoError> { sqlx::query_as::<_, ComponentRecord>( - "SELECT namespace, component_id, version, name, size, user_component, protected_component, protector_version, metadata FROM components WHERE namespace = $1 AND name = $2", + "SELECT components.namespace, components.name, component_versions.* FROM components JOIN component_versions ON components.component_id = component_versions.component_id WHERE components.namespace = $1 AND components.name = $2", ) .bind(namespace) .bind(name) @@ -270,22 +284,30 @@ impl ComponentRepo for DbComponentRepo { Ok(result.into_iter().map(|x| x.get("component_id")).collect()) } - async fn get_namespaces(&self, component_id: &Uuid) -> Result, RepoError> { - sqlx::query_as::<_, (String, i64)>( - "SELECT namespace, max(version) FROM components WHERE component_id = $1 GROUP BY namespace", - ) + async fn get_namespaces(&self, component_id: &Uuid) -> Result, RepoError> { + let result = sqlx::query("SELECT namespace FROM components WHERE component_id = $1") .bind(component_id) .fetch_all(self.db_pool.deref()) - .await - .map_err(|e| e.into()) + .await?; + + Ok(result.into_iter().map(|x| x.get("namespace")).collect()) } async fn delete(&self, namespace: &str, component_id: &Uuid) -> Result<(), RepoError> { + let mut transaction = self.db_pool.begin().await?; + sqlx::query("DELETE FROM component_versions WHERE component_id IN (SELECT component_id FROM components WHERE namespace = $1 AND component_id = $2)") + .bind(namespace) + .bind(component_id) + .execute(&mut *transaction) + .await?; + sqlx::query("DELETE FROM components WHERE namespace = $1 AND component_id = $2") .bind(namespace) .bind(component_id) - .execute(self.db_pool.deref()) + .execute(&mut *transaction) .await?; + + transaction.commit().await?; Ok(()) } } @@ -312,22 +334,34 @@ impl ComponentRepo for DbComponentRepo { sqlx::query( r#" INSERT INTO components - (namespace, component_id, version, name, size, user_component, protected_component, protector_version, metadata) + (namespace, component_id, name) + VALUES + ($1, $2, $3) + ON CONFLICT (namespace, name) DO NOTHING + "#, + ) + .bind(component.namespace.clone()) + .bind(component.component_id) + .bind(component.name.clone()) + .execute(&mut *transaction) + .await?; + + sqlx::query( + r#" + INSERT INTO component_versions + (component_id, version, size, user_component, protected_component, protector_version, metadata) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9) - ON CONFLICT (namespace, component_id, version) DO UPDATE - SET name = $4, - size = $5, - user_component = $6, - protected_component = $7, - protector_version = $8, - metadata = $9 + ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (component_id, version) DO UPDATE + SET size = $3, + user_component = $4, + protected_component = $5, + protector_version = $6, + metadata = $7 "#, ) - .bind(component.namespace.clone()) .bind(component.component_id) .bind(component.version) - .bind(component.name.clone()) .bind(component.size) .bind(component.user_component.clone()) .bind(component.protected_component.clone()) @@ -341,20 +375,14 @@ impl ComponentRepo for DbComponentRepo { Ok(()) } - async fn get_all(&self, namespace: &str) -> Result, RepoError> { - sqlx::query_as::<_, ComponentRecord>("SELECT namespace, component_id, name, size, version, user_component, protected_component, protector_version, metadata FROM components WHERE namespace = $1") - .bind(namespace) - .fetch_all(self.db_pool.deref()) - .await - .map_err(|e| e.into()) - } - async fn get( &self, namespace: &str, component_id: &Uuid, ) -> Result, RepoError> { - sqlx::query_as::<_, ComponentRecord>("SELECT namespace, component_id, name, size, version, user_component, protected_component, protector_version, metadata FROM components WHERE namespace = $1 AND component_id = $2") + sqlx::query_as::<_, ComponentRecord>( + "SELECT components.namespace, components.name, component_versions.* FROM components JOIN component_versions ON components.component_id = component_versions.component_id WHERE components.namespace = $1 AND component_versions.component_id = $2" + ) .bind(namespace) .bind(component_id) .fetch_all(self.db_pool.deref()) @@ -362,42 +390,22 @@ impl ComponentRepo for DbComponentRepo { .map_err(|e| e.into()) } - async fn get_by_name( - &self, - namespace: &str, - name: &str, - ) -> Result, RepoError> { - sqlx::query_as::<_, ComponentRecord>( - "SELECT namespace, component_id, name, size, version, user_component, protected_component, protector_version, metadata FROM components WHERE namespace = $1 AND name = $2", - ) + async fn get_all(&self, namespace: &str) -> Result, RepoError> { + sqlx::query_as::<_, ComponentRecord>("SELECT components.namespace, components.name, component_versions.* FROM components JOIN component_versions ON components.component_id = component_versions.component_id WHERE components.namespace = $1") .bind(namespace) - .bind(name) .fetch_all(self.db_pool.deref()) .await .map_err(|e| e.into()) } - async fn get_ids_by_name(&self, namespace: &str, name: &str) -> Result, RepoError> { - let result = sqlx::query( - "SELECT distinct component_id as component_id FROM components WHERE namespace = $1 AND name = $2" - ) - .bind(namespace) - .bind(name) - .fetch_all(self.db_pool.deref()) - .await?; - - Ok(result.into_iter().map(|x| x.get("component_id")).collect()) - } - async fn get_latest_version( &self, namespace: &str, component_id: &Uuid, ) -> Result, RepoError> { sqlx::query_as::<_, ComponentRecord>( - "SELECT namespace, component_id, name, size, version, user_component, protected_component, protector_version, metadata FROM components WHERE namespace = $1 AND component_id = $2 ORDER BY version DESC LIMIT 1", - ) - .bind(namespace) + "SELECT components.namespace, components.name, component_versions.* FROM components JOIN component_versions ON components.component_id = component_versions.component_id WHERE components.namespace = $1 AND component_versions.component_id = $2 ORDER BY version DESC LIMIT 1", + ).bind(namespace) .bind(component_id) .fetch_optional(self.db_pool.deref()) .await @@ -411,7 +419,7 @@ impl ComponentRepo for DbComponentRepo { version: u64, ) -> Result, RepoError> { sqlx::query_as::<_, ComponentRecord>( - "SELECT namespace, component_id, name, size, version, user_component, protected_component, protector_version, metadata FROM components WHERE namespace = $1 AND component_id = $2 AND version = $3", + "SELECT components.namespace, components.name, component_versions.* FROM components JOIN component_versions ON components.component_id = component_versions.component_id WHERE components.namespace = $1 AND component_versions.component_id = $2 AND component_versions.version = $3", ) .bind(namespace) .bind(component_id) @@ -421,22 +429,57 @@ impl ComponentRepo for DbComponentRepo { .map_err(|e| e.into()) } - async fn get_namespaces(&self, component_id: &Uuid) -> Result, RepoError> { - sqlx::query_as::<_, (String, i64)>( - "SELECT namespace, max(version) FROM components WHERE component_id = $1 GROUP BY namespace", + async fn get_by_name( + &self, + namespace: &str, + name: &str, + ) -> Result, RepoError> { + sqlx::query_as::<_, ComponentRecord>( + "SELECT components.namespace, components.name, component_versions.* FROM components JOIN component_versions ON components.component_id = component_versions.component_id WHERE components.namespace = $1 AND components.name = $2", ) - .bind(component_id) + .bind(namespace) + .bind(name) .fetch_all(self.db_pool.deref()) .await .map_err(|e| e.into()) } + async fn get_ids_by_name(&self, namespace: &str, name: &str) -> Result, RepoError> { + let result = sqlx::query( + "SELECT distinct component_id as component_id FROM components WHERE namespace = $1 AND name = $2" + ) + .bind(namespace) + .bind(name) + .fetch_all(self.db_pool.deref()) + .await?; + + Ok(result.into_iter().map(|x| x.get("component_id")).collect()) + } + + async fn get_namespaces(&self, component_id: &Uuid) -> Result, RepoError> { + let result = sqlx::query("SELECT namespace FROM components WHERE component_id = $1") + .bind(component_id) + .fetch_all(self.db_pool.deref()) + .await?; + + Ok(result.into_iter().map(|x| x.get("namespace")).collect()) + } + async fn delete(&self, namespace: &str, component_id: &Uuid) -> Result<(), RepoError> { + let mut transaction = self.db_pool.begin().await?; + sqlx::query("DELETE FROM component_versions WHERE component_id IN (SELECT component_id FROM components WHERE namespace = $1 AND component_id = $2)") + .bind(namespace) + .bind(component_id) + .execute(&mut *transaction) + .await?; + sqlx::query("DELETE FROM components WHERE namespace = $1 AND component_id = $2") .bind(namespace) .bind(component_id) - .execute(self.db_pool.deref()) + .execute(&mut *transaction) .await?; + + transaction.commit().await?; Ok(()) } } diff --git a/golem-component-service-base/src/service/component.rs b/golem-component-service-base/src/service/component.rs index e914f2d43b..0752956723 100644 --- a/golem-component-service-base/src/service/component.rs +++ b/golem-component-service-base/src/service/component.rs @@ -545,7 +545,7 @@ where if result.is_empty() { Ok(None) } else if result.len() == 1 { - let value = result[0].clone().0.try_into().map_err(|e| { + let value = result[0].clone().try_into().map_err(|e| { ComponentError::internal(e, "Failed to convert namespace".to_string()) })?; Ok(Some(value)) diff --git a/golem-component-service-base/tests/db/migration/postgres/001__init.sql b/golem-component-service-base/tests/db/migration/postgres/001__init.sql index fe3714fac1..be037afc2c 100644 --- a/golem-component-service-base/tests/db/migration/postgres/001__init.sql +++ b/golem-component-service-base/tests/db/migration/postgres/001__init.sql @@ -1,14 +1,23 @@ CREATE TABLE components ( namespace text NOT NULL, - component_id uuid NOT NULL, + component_id uuid NOT NULL PRIMARY KEY, name text NOT NULL, - size integer NOT NULL, + UNIQUE (namespace, name) +); + +CREATE INDEX components_namespace_id_idx ON components (namespace, component_id); + +CREATE TABLE component_versions +( + component_id uuid NOT NULL, version bigint NOT NULL, + size integer NOT NULL, created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, user_component text NOT NULL, protected_component text NOT NULL, protector_version bigint, metadata bytea NOT NULL, - PRIMARY KEY (namespace, component_id, version) + PRIMARY KEY (component_id, version), + FOREIGN KEY (component_id) REFERENCES components (component_id) ); diff --git a/golem-component-service-base/tests/db/migration/sqlite/001__init.sql b/golem-component-service-base/tests/db/migration/sqlite/001__init.sql index 673b3083df..222960279a 100644 --- a/golem-component-service-base/tests/db/migration/sqlite/001__init.sql +++ b/golem-component-service-base/tests/db/migration/sqlite/001__init.sql @@ -1,14 +1,24 @@ CREATE TABLE components ( namespace text NOT NULL, - component_id uuid NOT NULL, + component_id uuid NOT NULL PRIMARY KEY, name text NOT NULL, - size integer NOT NULL, + UNIQUE (namespace, name) +); + +CREATE INDEX components_namespace_id_idx ON components (namespace, component_id); + +CREATE TABLE component_versions +( + component_id uuid NOT NULL, version bigint NOT NULL, + size integer NOT NULL, created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, user_component text NOT NULL, protected_component text NOT NULL, protector_version bigint, metadata blob NOT NULL, - PRIMARY KEY (namespace, component_id, version) + PRIMARY KEY (component_id, version), + FOREIGN KEY (component_id) REFERENCES components (component_id) ); + diff --git a/golem-component-service-base/tests/services_tests.rs b/golem-component-service-base/tests/services_tests.rs index 3839a1bfec..6c2c4225ff 100644 --- a/golem-component-service-base/tests/services_tests.rs +++ b/golem-component-service-base/tests/services_tests.rs @@ -213,10 +213,40 @@ mod tests { assert_eq!(component1_result.unwrap(), DefaultNamespace::default()); let component2_result = component_service - .get_namespace(&component1.versioned_component_id.component_id) + .get_namespace(&component2.versioned_component_id.component_id) .await .unwrap(); assert!(component2_result.is_some()); assert_eq!(component2_result.unwrap(), DefaultNamespace::default()); + + let component1_result = component_service + .find_ids_by_name(&component1.component_name, &DefaultNamespace::default()) + .await + .unwrap(); + assert!(component1_result == vec![component1.versioned_component_id.component_id.clone()]); + + let component2_result = component_service + .find_ids_by_name(&component2.component_name, &DefaultNamespace::default()) + .await + .unwrap(); + assert!(component2_result == vec![component2.versioned_component_id.component_id.clone()]); + + let component1_result = component_service + .find_by_name( + Some(component1.component_name.clone()), + &DefaultNamespace::default(), + ) + .await + .unwrap(); + assert!(component1_result == vec![component1.clone(), component1v2.clone()]); + + let component2_result = component_service + .find_by_name( + Some(component2.component_name.clone()), + &DefaultNamespace::default(), + ) + .await + .unwrap(); + assert!(component2_result == vec![component2.clone()]); } } diff --git a/golem-component-service/db/migration/postgres/001__init.sql b/golem-component-service/db/migration/postgres/001__init.sql index fe3714fac1..be037afc2c 100644 --- a/golem-component-service/db/migration/postgres/001__init.sql +++ b/golem-component-service/db/migration/postgres/001__init.sql @@ -1,14 +1,23 @@ CREATE TABLE components ( namespace text NOT NULL, - component_id uuid NOT NULL, + component_id uuid NOT NULL PRIMARY KEY, name text NOT NULL, - size integer NOT NULL, + UNIQUE (namespace, name) +); + +CREATE INDEX components_namespace_id_idx ON components (namespace, component_id); + +CREATE TABLE component_versions +( + component_id uuid NOT NULL, version bigint NOT NULL, + size integer NOT NULL, created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, user_component text NOT NULL, protected_component text NOT NULL, protector_version bigint, metadata bytea NOT NULL, - PRIMARY KEY (namespace, component_id, version) + PRIMARY KEY (component_id, version), + FOREIGN KEY (component_id) REFERENCES components (component_id) ); diff --git a/golem-component-service/db/migration/sqlite/001__init.sql b/golem-component-service/db/migration/sqlite/001__init.sql index 673b3083df..222960279a 100644 --- a/golem-component-service/db/migration/sqlite/001__init.sql +++ b/golem-component-service/db/migration/sqlite/001__init.sql @@ -1,14 +1,24 @@ CREATE TABLE components ( namespace text NOT NULL, - component_id uuid NOT NULL, + component_id uuid NOT NULL PRIMARY KEY, name text NOT NULL, - size integer NOT NULL, + UNIQUE (namespace, name) +); + +CREATE INDEX components_namespace_id_idx ON components (namespace, component_id); + +CREATE TABLE component_versions +( + component_id uuid NOT NULL, version bigint NOT NULL, + size integer NOT NULL, created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, user_component text NOT NULL, protected_component text NOT NULL, protector_version bigint, metadata blob NOT NULL, - PRIMARY KEY (namespace, component_id, version) + PRIMARY KEY (component_id, version), + FOREIGN KEY (component_id) REFERENCES components (component_id) ); + From 0e21ac2e8147372c1e56622647b77a58184e38c8 Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Tue, 2 Jul 2024 22:13:39 +0200 Subject: [PATCH 14/30] component-service get_id_by_name, get_namespace --- .../src/repo/component.rs | 50 +++++++++---------- .../src/service/component.rs | 34 ++++++------- .../tests/db/migration/postgres/001__init.sql | 7 ++- .../tests/db/migration/sqlite/001__init.sql | 8 ++- .../tests/services_tests.rs | 8 +-- .../db/migration/postgres/001__init.sql | 7 ++- .../db/migration/sqlite/001__init.sql | 8 ++- 7 files changed, 54 insertions(+), 68 deletions(-) diff --git a/golem-component-service-base/src/repo/component.rs b/golem-component-service-base/src/repo/component.rs index ff55ae0036..4093b17c7b 100644 --- a/golem-component-service-base/src/repo/component.rs +++ b/golem-component-service-base/src/repo/component.rs @@ -123,9 +123,9 @@ pub trait ComponentRepo { name: &str, ) -> Result, RepoError>; - async fn get_ids_by_name(&self, namespace: &str, name: &str) -> Result, RepoError>; + async fn get_id_by_name(&self, namespace: &str, name: &str) -> Result, RepoError>; - async fn get_namespaces(&self, component_id: &Uuid) -> Result, RepoError>; + async fn get_namespace(&self, component_id: &Uuid) -> Result, RepoError>; async fn delete(&self, namespace: &str, component_id: &Uuid) -> Result<(), RepoError>; } @@ -272,25 +272,24 @@ impl ComponentRepo for DbComponentRepo { .map_err(|e| e.into()) } - async fn get_ids_by_name(&self, namespace: &str, name: &str) -> Result, RepoError> { - let result = sqlx::query( - "SELECT distinct component_id as component_id FROM components WHERE namespace = $1 AND name = $2" - ) - .bind(namespace) - .bind(name) - .fetch_all(self.db_pool.deref()) - .await?; + async fn get_id_by_name(&self, namespace: &str, name: &str) -> Result, RepoError> { + let result = + sqlx::query("SELECT component_id FROM components WHERE namespace = $1 AND name = $2") + .bind(namespace) + .bind(name) + .fetch_optional(self.db_pool.deref()) + .await?; - Ok(result.into_iter().map(|x| x.get("component_id")).collect()) + Ok(result.map(|x| x.get("component_id"))) } - async fn get_namespaces(&self, component_id: &Uuid) -> Result, RepoError> { + async fn get_namespace(&self, component_id: &Uuid) -> Result, RepoError> { let result = sqlx::query("SELECT namespace FROM components WHERE component_id = $1") .bind(component_id) - .fetch_all(self.db_pool.deref()) + .fetch_optional(self.db_pool.deref()) .await?; - Ok(result.into_iter().map(|x| x.get("namespace")).collect()) + Ok(result.map(|x| x.get("namespace"))) } async fn delete(&self, namespace: &str, component_id: &Uuid) -> Result<(), RepoError> { @@ -444,25 +443,24 @@ impl ComponentRepo for DbComponentRepo { .map_err(|e| e.into()) } - async fn get_ids_by_name(&self, namespace: &str, name: &str) -> Result, RepoError> { - let result = sqlx::query( - "SELECT distinct component_id as component_id FROM components WHERE namespace = $1 AND name = $2" - ) - .bind(namespace) - .bind(name) - .fetch_all(self.db_pool.deref()) - .await?; + async fn get_id_by_name(&self, namespace: &str, name: &str) -> Result, RepoError> { + let result = + sqlx::query("SELECT component_id FROM components WHERE namespace = $1 AND name = $2") + .bind(namespace) + .bind(name) + .fetch_optional(self.db_pool.deref()) + .await?; - Ok(result.into_iter().map(|x| x.get("component_id")).collect()) + Ok(result.map(|x| x.get("component_id"))) } - async fn get_namespaces(&self, component_id: &Uuid) -> Result, RepoError> { + async fn get_namespace(&self, component_id: &Uuid) -> Result, RepoError> { let result = sqlx::query("SELECT namespace FROM components WHERE component_id = $1") .bind(component_id) - .fetch_all(self.db_pool.deref()) + .fetch_optional(self.db_pool.deref()) .await?; - Ok(result.into_iter().map(|x| x.get("namespace")).collect()) + Ok(result.map(|x| x.get("namespace"))) } async fn delete(&self, namespace: &str, component_id: &Uuid) -> Result<(), RepoError> { diff --git a/golem-component-service-base/src/service/component.rs b/golem-component-service-base/src/service/component.rs index 0752956723..3b0e177a88 100644 --- a/golem-component-service-base/src/service/component.rs +++ b/golem-component-service-base/src/service/component.rs @@ -102,11 +102,11 @@ pub trait ComponentService { namespace: &Namespace, ) -> Result, ComponentError>; - async fn find_ids_by_name( + async fn find_id_by_name( &self, component_name: &ComponentName, namespace: &Namespace, - ) -> Result, ComponentError>; + ) -> Result, ComponentError>; async fn get_by_version( &self, @@ -172,7 +172,7 @@ where component_name.0.clone() ); - self.find_ids_by_name(component_name, namespace) + self.find_id_by_name(component_name, namespace) .await? .into_iter() .next() @@ -412,16 +412,16 @@ where Ok(Some(result)) } - async fn find_ids_by_name( + async fn find_id_by_name( &self, component_name: &ComponentName, namespace: &Namespace, - ) -> Result, ComponentError> { + ) -> Result, ComponentError> { let records = self .component_repo - .get_ids_by_name(namespace.to_string().as_str(), &component_name.0) + .get_id_by_name(namespace.to_string().as_str(), &component_name.0) .await?; - Ok(records.into_iter().map(ComponentId).collect()) + Ok(records.map(ComponentId)) } async fn find_by_name( @@ -540,20 +540,14 @@ where component_id: &ComponentId, ) -> Result, ComponentError> { info!("Getting component namespace - id: {}", component_id); - let result = self.component_repo.get_namespaces(&component_id.0).await?; - - if result.is_empty() { - Ok(None) - } else if result.len() == 1 { - let value = result[0].clone().try_into().map_err(|e| { + let result = self.component_repo.get_namespace(&component_id.0).await?; + if let Some(result) = result { + let value = result.clone().try_into().map_err(|e| { ComponentError::internal(e, "Failed to convert namespace".to_string()) })?; Ok(Some(value)) } else { - Err(ComponentError::internal( - "", - "Namespace is not unique".to_string(), - )) + Ok(None) } } } @@ -701,12 +695,12 @@ impl + Eq + Clone + Send + Sync> ComponentS Ok(ByteStream::empty()) } - async fn find_ids_by_name( + async fn find_id_by_name( &self, _component_name: &ComponentName, _namespace: &Namespace, - ) -> Result, ComponentError> { - Ok(vec![]) + ) -> Result, ComponentError> { + Ok(None) } async fn get_protected_data( diff --git a/golem-component-service-base/tests/db/migration/postgres/001__init.sql b/golem-component-service-base/tests/db/migration/postgres/001__init.sql index be037afc2c..d15a9fa3f4 100644 --- a/golem-component-service-base/tests/db/migration/postgres/001__init.sql +++ b/golem-component-service-base/tests/db/migration/postgres/001__init.sql @@ -1,7 +1,7 @@ CREATE TABLE components ( - namespace text NOT NULL, component_id uuid NOT NULL PRIMARY KEY, + namespace text NOT NULL, name text NOT NULL, UNIQUE (namespace, name) ); @@ -10,7 +10,7 @@ CREATE INDEX components_namespace_id_idx ON components (namespace, component_id) CREATE TABLE component_versions ( - component_id uuid NOT NULL, + component_id uuid NOT NULL REFERENCES components (component_id), version bigint NOT NULL, size integer NOT NULL, created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, @@ -18,6 +18,5 @@ CREATE TABLE component_versions protected_component text NOT NULL, protector_version bigint, metadata bytea NOT NULL, - PRIMARY KEY (component_id, version), - FOREIGN KEY (component_id) REFERENCES components (component_id) + PRIMARY KEY (component_id, version) ); diff --git a/golem-component-service-base/tests/db/migration/sqlite/001__init.sql b/golem-component-service-base/tests/db/migration/sqlite/001__init.sql index 222960279a..e1bb4c27ae 100644 --- a/golem-component-service-base/tests/db/migration/sqlite/001__init.sql +++ b/golem-component-service-base/tests/db/migration/sqlite/001__init.sql @@ -1,7 +1,7 @@ CREATE TABLE components ( - namespace text NOT NULL, component_id uuid NOT NULL PRIMARY KEY, + namespace text NOT NULL, name text NOT NULL, UNIQUE (namespace, name) ); @@ -10,7 +10,7 @@ CREATE INDEX components_namespace_id_idx ON components (namespace, component_id) CREATE TABLE component_versions ( - component_id uuid NOT NULL, + component_id uuid NOT NULL REFERENCES components (component_id), version bigint NOT NULL, size integer NOT NULL, created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, @@ -18,7 +18,5 @@ CREATE TABLE component_versions protected_component text NOT NULL, protector_version bigint, metadata blob NOT NULL, - PRIMARY KEY (component_id, version), - FOREIGN KEY (component_id) REFERENCES components (component_id) + PRIMARY KEY (component_id, version) ); - diff --git a/golem-component-service-base/tests/services_tests.rs b/golem-component-service-base/tests/services_tests.rs index 6c2c4225ff..9ff61c4ebf 100644 --- a/golem-component-service-base/tests/services_tests.rs +++ b/golem-component-service-base/tests/services_tests.rs @@ -220,16 +220,16 @@ mod tests { assert_eq!(component2_result.unwrap(), DefaultNamespace::default()); let component1_result = component_service - .find_ids_by_name(&component1.component_name, &DefaultNamespace::default()) + .find_id_by_name(&component1.component_name, &DefaultNamespace::default()) .await .unwrap(); - assert!(component1_result == vec![component1.versioned_component_id.component_id.clone()]); + assert!(component1_result == Some(component1.versioned_component_id.component_id.clone())); let component2_result = component_service - .find_ids_by_name(&component2.component_name, &DefaultNamespace::default()) + .find_id_by_name(&component2.component_name, &DefaultNamespace::default()) .await .unwrap(); - assert!(component2_result == vec![component2.versioned_component_id.component_id.clone()]); + assert!(component2_result == Some(component2.versioned_component_id.component_id.clone())); let component1_result = component_service .find_by_name( diff --git a/golem-component-service/db/migration/postgres/001__init.sql b/golem-component-service/db/migration/postgres/001__init.sql index be037afc2c..d15a9fa3f4 100644 --- a/golem-component-service/db/migration/postgres/001__init.sql +++ b/golem-component-service/db/migration/postgres/001__init.sql @@ -1,7 +1,7 @@ CREATE TABLE components ( - namespace text NOT NULL, component_id uuid NOT NULL PRIMARY KEY, + namespace text NOT NULL, name text NOT NULL, UNIQUE (namespace, name) ); @@ -10,7 +10,7 @@ CREATE INDEX components_namespace_id_idx ON components (namespace, component_id) CREATE TABLE component_versions ( - component_id uuid NOT NULL, + component_id uuid NOT NULL REFERENCES components (component_id), version bigint NOT NULL, size integer NOT NULL, created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, @@ -18,6 +18,5 @@ CREATE TABLE component_versions protected_component text NOT NULL, protector_version bigint, metadata bytea NOT NULL, - PRIMARY KEY (component_id, version), - FOREIGN KEY (component_id) REFERENCES components (component_id) + PRIMARY KEY (component_id, version) ); diff --git a/golem-component-service/db/migration/sqlite/001__init.sql b/golem-component-service/db/migration/sqlite/001__init.sql index 222960279a..e1bb4c27ae 100644 --- a/golem-component-service/db/migration/sqlite/001__init.sql +++ b/golem-component-service/db/migration/sqlite/001__init.sql @@ -1,7 +1,7 @@ CREATE TABLE components ( - namespace text NOT NULL, component_id uuid NOT NULL PRIMARY KEY, + namespace text NOT NULL, name text NOT NULL, UNIQUE (namespace, name) ); @@ -10,7 +10,7 @@ CREATE INDEX components_namespace_id_idx ON components (namespace, component_id) CREATE TABLE component_versions ( - component_id uuid NOT NULL, + component_id uuid NOT NULL REFERENCES components (component_id), version bigint NOT NULL, size integer NOT NULL, created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, @@ -18,7 +18,5 @@ CREATE TABLE component_versions protected_component text NOT NULL, protector_version bigint, metadata blob NOT NULL, - PRIMARY KEY (component_id, version), - FOREIGN KEY (component_id) REFERENCES components (component_id) + PRIMARY KEY (component_id, version) ); - From e6cdf752761ed935e376e203b1dd720c3a1ed45c Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Tue, 2 Jul 2024 22:38:48 +0200 Subject: [PATCH 15/30] component-repo queries with specific column names --- .../src/repo/component.rs | 118 +++++++++++++++--- .../src/service/component.rs | 2 - .../tests/services_tests.rs | 6 + 3 files changed, 104 insertions(+), 22 deletions(-) diff --git a/golem-component-service-base/src/repo/component.rs b/golem-component-service-base/src/repo/component.rs index 4093b17c7b..b1889ff910 100644 --- a/golem-component-service-base/src/repo/component.rs +++ b/golem-component-service-base/src/repo/component.rs @@ -148,10 +148,10 @@ impl ComponentRepo for DbComponentRepo { let result = sqlx::query( "SELECT count(component_id) as count FROM components WHERE namespace != $1 AND component_id = $2", ) - .bind(component.namespace.clone()) - .bind(component.component_id) - .fetch_one(&mut *transaction) - .await?; + .bind(component.namespace.clone()) + .bind(component.component_id) + .fetch_one(&mut *transaction) + .await?; let count: i64 = result.get("count"); @@ -209,7 +209,14 @@ impl ComponentRepo for DbComponentRepo { component_id: &Uuid, ) -> Result, RepoError> { sqlx::query_as::<_, ComponentRecord>( - "SELECT components.namespace, components.name, component_versions.* FROM components JOIN component_versions ON components.component_id = component_versions.component_id WHERE components.namespace = $1 AND component_versions.component_id = $2" + r#" + SELECT c.namespace AS namespace, c.name AS name, c.component_id AS component_id, + cv.version AS version, cv.size AS size, cv.user_component AS user_component, + cv.protected_component AS protected_component, cv.protector_version AS protector_version, + cv.metadata AS metadata + FROM components c JOIN component_versions cv ON c.component_id = cv.component_id + WHERE c.namespace = $1 AND c.component_id = $2 + "# ) .bind(namespace) .bind(component_id) @@ -219,7 +226,16 @@ impl ComponentRepo for DbComponentRepo { } async fn get_all(&self, namespace: &str) -> Result, RepoError> { - sqlx::query_as::<_, ComponentRecord>("SELECT components.namespace, components.name, component_versions.* FROM components JOIN component_versions ON components.component_id = component_versions.component_id WHERE components.namespace = $1") + sqlx::query_as::<_, ComponentRecord>( + r#" + SELECT c.namespace AS namespace, c.name AS name, c.component_id AS component_id, + cv.version AS version, cv.size AS size, cv.user_component AS user_component, + cv.protected_component AS protected_component, cv.protector_version AS protector_version, + cv.metadata AS metadata + FROM components c JOIN component_versions cv ON c.component_id = cv.component_id + WHERE c.namespace = $1 + "# + ) .bind(namespace) .fetch_all(self.db_pool.deref()) .await @@ -232,8 +248,17 @@ impl ComponentRepo for DbComponentRepo { component_id: &Uuid, ) -> Result, RepoError> { sqlx::query_as::<_, ComponentRecord>( - "SELECT components.namespace, components.name, component_versions.* FROM components JOIN component_versions ON components.component_id = component_versions.component_id WHERE components.namespace = $1 AND component_versions.component_id = $2 ORDER BY version DESC LIMIT 1", - ).bind(namespace) + r#" + SELECT c.namespace AS namespace, c.name AS name, c.component_id AS component_id, + cv.version AS version, cv.size AS size, cv.user_component AS user_component, + cv.protected_component AS protected_component, cv.protector_version AS protector_version, + cv.metadata AS metadata + FROM components c JOIN component_versions cv ON c.component_id = cv.component_id + WHERE c.namespace = $1 AND c.component_id = $2 + ORDER BY cv.version DESC LIMIT 1 + "#, + ) + .bind(namespace) .bind(component_id) .fetch_optional(self.db_pool.deref()) .await @@ -247,7 +272,14 @@ impl ComponentRepo for DbComponentRepo { version: u64, ) -> Result, RepoError> { sqlx::query_as::<_, ComponentRecord>( - "SELECT components.namespace, components.name, component_versions.* FROM components JOIN component_versions ON components.component_id = component_versions.component_id WHERE components.namespace = $1 AND component_versions.component_id = $2 AND component_versions.version = $3", + r#" + SELECT c.namespace AS namespace, c.name AS name, c.component_id AS component_id, + cv.version AS version, cv.size AS size, cv.user_component AS user_component, + cv.protected_component AS protected_component, cv.protector_version AS protector_version, + cv.metadata AS metadata + FROM components c JOIN component_versions cv ON c.component_id = cv.component_id + WHERE c.namespace = $1 AND c.component_id = $2 AND cv.version = $3 + "#, ) .bind(namespace) .bind(component_id) @@ -263,7 +295,14 @@ impl ComponentRepo for DbComponentRepo { name: &str, ) -> Result, RepoError> { sqlx::query_as::<_, ComponentRecord>( - "SELECT components.namespace, components.name, component_versions.* FROM components JOIN component_versions ON components.component_id = component_versions.component_id WHERE components.namespace = $1 AND components.name = $2", + r#" + SELECT c.namespace AS namespace, c.name AS name, c.component_id AS component_id, + cv.version AS version, cv.size AS size, cv.user_component AS user_component, + cv.protected_component AS protected_component, cv.protector_version AS protector_version, + cv.metadata AS metadata + FROM components c JOIN component_versions cv ON c.component_id = cv.component_id + WHERE c.namespace = $1 AND c.name = $2 + "#, ) .bind(namespace) .bind(name) @@ -319,10 +358,10 @@ impl ComponentRepo for DbComponentRepo { let result = sqlx::query( "SELECT count(component_id) as count FROM components WHERE namespace != $1 AND component_id = $2", ) - .bind(component.namespace.clone()) - .bind(component.component_id) - .fetch_one(&mut *transaction) - .await?; + .bind(component.namespace.clone()) + .bind(component.component_id) + .fetch_one(&mut *transaction) + .await?; let count: i64 = result.get("count"); @@ -380,7 +419,14 @@ impl ComponentRepo for DbComponentRepo { component_id: &Uuid, ) -> Result, RepoError> { sqlx::query_as::<_, ComponentRecord>( - "SELECT components.namespace, components.name, component_versions.* FROM components JOIN component_versions ON components.component_id = component_versions.component_id WHERE components.namespace = $1 AND component_versions.component_id = $2" + r#" + SELECT c.namespace AS namespace, c.name AS name, c.component_id AS component_id, + cv.version AS version, cv.size AS size, cv.user_component AS user_component, + cv.protected_component AS protected_component, cv.protector_version AS protector_version, + cv.metadata AS metadata + FROM components c JOIN component_versions cv ON c.component_id = cv.component_id + WHERE c.namespace = $1 AND c.component_id = $2 + "# ) .bind(namespace) .bind(component_id) @@ -390,7 +436,16 @@ impl ComponentRepo for DbComponentRepo { } async fn get_all(&self, namespace: &str) -> Result, RepoError> { - sqlx::query_as::<_, ComponentRecord>("SELECT components.namespace, components.name, component_versions.* FROM components JOIN component_versions ON components.component_id = component_versions.component_id WHERE components.namespace = $1") + sqlx::query_as::<_, ComponentRecord>( + r#" + SELECT c.namespace AS namespace, c.name AS name, c.component_id AS component_id, + cv.version AS version, cv.size AS size, cv.user_component AS user_component, + cv.protected_component AS protected_component, cv.protector_version AS protector_version, + cv.metadata AS metadata + FROM components c JOIN component_versions cv ON c.component_id = cv.component_id + WHERE c.namespace = $1 + "# + ) .bind(namespace) .fetch_all(self.db_pool.deref()) .await @@ -403,8 +458,17 @@ impl ComponentRepo for DbComponentRepo { component_id: &Uuid, ) -> Result, RepoError> { sqlx::query_as::<_, ComponentRecord>( - "SELECT components.namespace, components.name, component_versions.* FROM components JOIN component_versions ON components.component_id = component_versions.component_id WHERE components.namespace = $1 AND component_versions.component_id = $2 ORDER BY version DESC LIMIT 1", - ).bind(namespace) + r#" + SELECT c.namespace AS namespace, c.name AS name, c.component_id AS component_id, + cv.version AS version, cv.size AS size, cv.user_component AS user_component, + cv.protected_component AS protected_component, cv.protector_version AS protector_version, + cv.metadata AS metadata + FROM components c JOIN component_versions cv ON c.component_id = cv.component_id + WHERE c.namespace = $1 AND c.component_id = $2 + ORDER BY cv.version DESC LIMIT 1 + "#, + ) + .bind(namespace) .bind(component_id) .fetch_optional(self.db_pool.deref()) .await @@ -418,7 +482,14 @@ impl ComponentRepo for DbComponentRepo { version: u64, ) -> Result, RepoError> { sqlx::query_as::<_, ComponentRecord>( - "SELECT components.namespace, components.name, component_versions.* FROM components JOIN component_versions ON components.component_id = component_versions.component_id WHERE components.namespace = $1 AND component_versions.component_id = $2 AND component_versions.version = $3", + r#" + SELECT c.namespace AS namespace, c.name AS name, c.component_id AS component_id, + cv.version AS version, cv.size AS size, cv.user_component AS user_component, + cv.protected_component AS protected_component, cv.protector_version AS protector_version, + cv.metadata AS metadata + FROM components c JOIN component_versions cv ON c.component_id = cv.component_id + WHERE c.namespace = $1 AND c.component_id = $2 AND cv.version = $3 + "#, ) .bind(namespace) .bind(component_id) @@ -434,7 +505,14 @@ impl ComponentRepo for DbComponentRepo { name: &str, ) -> Result, RepoError> { sqlx::query_as::<_, ComponentRecord>( - "SELECT components.namespace, components.name, component_versions.* FROM components JOIN component_versions ON components.component_id = component_versions.component_id WHERE components.namespace = $1 AND components.name = $2", + r#" + SELECT c.namespace AS namespace, c.name AS name, c.component_id AS component_id, + cv.version AS version, cv.size AS size, cv.user_component AS user_component, + cv.protected_component AS protected_component, cv.protector_version AS protector_version, + cv.metadata AS metadata + FROM components c JOIN component_versions cv ON c.component_id = cv.component_id + WHERE c.namespace = $1 AND c.name = $2 + "#, ) .bind(namespace) .bind(name) diff --git a/golem-component-service-base/src/service/component.rs b/golem-component-service-base/src/service/component.rs index 3b0e177a88..94253b8670 100644 --- a/golem-component-service-base/src/service/component.rs +++ b/golem-component-service-base/src/service/component.rs @@ -174,8 +174,6 @@ where self.find_id_by_name(component_name, namespace) .await? - .into_iter() - .next() .map_or(Ok(()), |id| Err(ComponentError::AlreadyExists(id)))?; let metadata = process_component(&data)?; diff --git a/golem-component-service-base/tests/services_tests.rs b/golem-component-service-base/tests/services_tests.rs index 9ff61c4ebf..230c3999df 100644 --- a/golem-component-service-base/tests/services_tests.rs +++ b/golem-component-service-base/tests/services_tests.rs @@ -248,5 +248,11 @@ mod tests { .await .unwrap(); assert!(component2_result == vec![component2.clone()]); + + let component_result = component_service + .find_by_name(None, &DefaultNamespace::default()) + .await + .unwrap(); + assert!(component_result.len() == 3); } } From 4ccf0f3d8caff2b464b5413d316cee2af24c0da6 Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Tue, 2 Jul 2024 23:09:18 +0200 Subject: [PATCH 16/30] component-repo upsert -> create --- .../src/repo/component.rs | 108 ++++++++---------- .../src/service/component.rs | 4 +- .../tests/db/migration/postgres/001__init.sql | 5 +- .../tests/db/migration/sqlite/001__init.sql | 5 +- .../db/migration/postgres/001__init.sql | 5 +- .../db/migration/sqlite/001__init.sql | 5 +- 6 files changed, 59 insertions(+), 73 deletions(-) diff --git a/golem-component-service-base/src/repo/component.rs b/golem-component-service-base/src/repo/component.rs index b1889ff910..07415c397c 100644 --- a/golem-component-service-base/src/repo/component.rs +++ b/golem-component-service-base/src/repo/component.rs @@ -94,7 +94,7 @@ impl ComponentRecord { #[async_trait] pub trait ComponentRepo { - async fn upsert(&self, component: &ComponentRecord) -> Result<(), RepoError>; + async fn create(&self, component: &ComponentRecord) -> Result<(), RepoError>; async fn get( &self, @@ -142,50 +142,41 @@ impl DbComponentRepo { #[async_trait] impl ComponentRepo for DbComponentRepo { - async fn upsert(&self, component: &ComponentRecord) -> Result<(), RepoError> { + async fn create(&self, component: &ComponentRecord) -> Result<(), RepoError> { let mut transaction = self.db_pool.begin().await?; - let result = sqlx::query( - "SELECT count(component_id) as count FROM components WHERE namespace != $1 AND component_id = $2", - ) - .bind(component.namespace.clone()) + let result = sqlx::query("SELECT namespace FROM components WHERE component_id = $1") .bind(component.component_id) - .fetch_one(&mut *transaction) + .fetch_optional(&mut *transaction) .await?; - let count: i64 = result.get("count"); - - if count > 0 { - return Err(RepoError::Internal("Component id not unique".to_string())); + if let Some(result) = result { + let namespace: String = result.get("namespace"); + if namespace != component.namespace { + return Err(RepoError::Internal("Component id not unique".to_string())); + } + } else { + sqlx::query( + r#" + INSERT INTO components + (namespace, component_id, name) + VALUES + ($1, $2, $3) + "#, + ) + .bind(component.namespace.clone()) + .bind(component.component_id) + .bind(component.name.clone()) + .execute(&mut *transaction) + .await?; } - sqlx::query( - r#" - INSERT INTO components - (namespace, component_id, name) - VALUES - ($1, $2, $3) - ON CONFLICT (namespace, name) DO NOTHING - "#, - ) - .bind(component.namespace.clone()) - .bind(component.component_id) - .bind(component.name.clone()) - .execute(&mut *transaction) - .await?; - sqlx::query( r#" INSERT INTO component_versions (component_id, version, size, user_component, protected_component, protector_version, metadata) VALUES ($1, $2, $3, $4, $5, $6, $7) - ON CONFLICT (component_id, version) DO UPDATE - SET size = $3, - user_component = $4, - protected_component = $5, - protector_version = $6, - metadata = $7 "#, ) .bind(component.component_id) @@ -352,50 +343,41 @@ impl ComponentRepo for DbComponentRepo { #[async_trait] impl ComponentRepo for DbComponentRepo { - async fn upsert(&self, component: &ComponentRecord) -> Result<(), RepoError> { + async fn create(&self, component: &ComponentRecord) -> Result<(), RepoError> { let mut transaction = self.db_pool.begin().await?; - let result = sqlx::query( - "SELECT count(component_id) as count FROM components WHERE namespace != $1 AND component_id = $2", - ) - .bind(component.namespace.clone()) + let result = sqlx::query("SELECT namespace FROM components WHERE component_id = $1") .bind(component.component_id) - .fetch_one(&mut *transaction) + .fetch_optional(&mut *transaction) .await?; - let count: i64 = result.get("count"); - - if count > 0 { - return Err(RepoError::Internal("Component id not unique".to_string())); + if let Some(result) = result { + let namespace: String = result.get("namespace"); + if namespace != component.namespace { + return Err(RepoError::Internal("Component id not unique".to_string())); + } + } else { + sqlx::query( + r#" + INSERT INTO components + (namespace, component_id, name) + VALUES + ($1, $2, $3) + "#, + ) + .bind(component.namespace.clone()) + .bind(component.component_id) + .bind(component.name.clone()) + .execute(&mut *transaction) + .await?; } - sqlx::query( - r#" - INSERT INTO components - (namespace, component_id, name) - VALUES - ($1, $2, $3) - ON CONFLICT (namespace, name) DO NOTHING - "#, - ) - .bind(component.namespace.clone()) - .bind(component.component_id) - .bind(component.name.clone()) - .execute(&mut *transaction) - .await?; - sqlx::query( r#" INSERT INTO component_versions (component_id, version, size, user_component, protected_component, protector_version, metadata) VALUES ($1, $2, $3, $4, $5, $6, $7) - ON CONFLICT (component_id, version) DO UPDATE - SET size = $3, - user_component = $4, - protected_component = $5, - protector_version = $6, - metadata = $7 "#, ) .bind(component.component_id) diff --git a/golem-component-service-base/src/service/component.rs b/golem-component-service-base/src/service/component.rs index 94253b8670..429c1087c1 100644 --- a/golem-component-service-base/src/service/component.rs +++ b/golem-component-service-base/src/service/component.rs @@ -217,7 +217,7 @@ where let record = ComponentRecord::new(namespace, component.clone()) .map_err(|e| ComponentError::internal(e, "Failed to convert record"))?; - self.component_repo.upsert(&record).await?; + self.component_repo.create(&record).await?; self.component_compilation .enqueue_compilation(&component.versioned_component_id.component_id, 0) @@ -276,7 +276,7 @@ where let record = ComponentRecord::new(namespace, component.clone()) .map_err(|e| ComponentError::internal(e, "Failed to convert record"))?; - self.component_repo.upsert(&record).await?; + self.component_repo.create(&record).await?; self.component_compilation .enqueue_compilation(component_id, component.versioned_component_id.version) diff --git a/golem-component-service-base/tests/db/migration/postgres/001__init.sql b/golem-component-service-base/tests/db/migration/postgres/001__init.sql index d15a9fa3f4..5c8f7dea1c 100644 --- a/golem-component-service-base/tests/db/migration/postgres/001__init.sql +++ b/golem-component-service-base/tests/db/migration/postgres/001__init.sql @@ -2,12 +2,13 @@ CREATE TABLE components ( component_id uuid NOT NULL PRIMARY KEY, namespace text NOT NULL, - name text NOT NULL, - UNIQUE (namespace, name) + name text NOT NULL ); CREATE INDEX components_namespace_id_idx ON components (namespace, component_id); +CREATE UNIQUE INDEX components_namespace_name_idx ON components (namespace, name); + CREATE TABLE component_versions ( component_id uuid NOT NULL REFERENCES components (component_id), diff --git a/golem-component-service-base/tests/db/migration/sqlite/001__init.sql b/golem-component-service-base/tests/db/migration/sqlite/001__init.sql index e1bb4c27ae..0bb14b9dcb 100644 --- a/golem-component-service-base/tests/db/migration/sqlite/001__init.sql +++ b/golem-component-service-base/tests/db/migration/sqlite/001__init.sql @@ -2,12 +2,13 @@ CREATE TABLE components ( component_id uuid NOT NULL PRIMARY KEY, namespace text NOT NULL, - name text NOT NULL, - UNIQUE (namespace, name) + name text NOT NULL ); CREATE INDEX components_namespace_id_idx ON components (namespace, component_id); +CREATE UNIQUE INDEX components_namespace_name_idx ON components (namespace, name); + CREATE TABLE component_versions ( component_id uuid NOT NULL REFERENCES components (component_id), diff --git a/golem-component-service/db/migration/postgres/001__init.sql b/golem-component-service/db/migration/postgres/001__init.sql index d15a9fa3f4..5c8f7dea1c 100644 --- a/golem-component-service/db/migration/postgres/001__init.sql +++ b/golem-component-service/db/migration/postgres/001__init.sql @@ -2,12 +2,13 @@ CREATE TABLE components ( component_id uuid NOT NULL PRIMARY KEY, namespace text NOT NULL, - name text NOT NULL, - UNIQUE (namespace, name) + name text NOT NULL ); CREATE INDEX components_namespace_id_idx ON components (namespace, component_id); +CREATE UNIQUE INDEX components_namespace_name_idx ON components (namespace, name); + CREATE TABLE component_versions ( component_id uuid NOT NULL REFERENCES components (component_id), diff --git a/golem-component-service/db/migration/sqlite/001__init.sql b/golem-component-service/db/migration/sqlite/001__init.sql index e1bb4c27ae..0bb14b9dcb 100644 --- a/golem-component-service/db/migration/sqlite/001__init.sql +++ b/golem-component-service/db/migration/sqlite/001__init.sql @@ -2,12 +2,13 @@ CREATE TABLE components ( component_id uuid NOT NULL PRIMARY KEY, namespace text NOT NULL, - name text NOT NULL, - UNIQUE (namespace, name) + name text NOT NULL ); CREATE INDEX components_namespace_id_idx ON components (namespace, component_id); +CREATE UNIQUE INDEX components_namespace_name_idx ON components (namespace, name); + CREATE TABLE component_versions ( component_id uuid NOT NULL REFERENCES components (component_id), From 345ba2209d37a7a711e54c696b3e4761dbbd1fc1 Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Wed, 3 Jul 2024 10:10:02 +0200 Subject: [PATCH 17/30] component-repo sql format, tests --- .../src/repo/component.rs | 308 +++++++++++------- .../src/service/component.rs | 66 ++-- .../tests/services_tests.rs | 128 +++++++- 3 files changed, 347 insertions(+), 155 deletions(-) diff --git a/golem-component-service-base/src/repo/component.rs b/golem-component-service-base/src/repo/component.rs index 07415c397c..f208541e48 100644 --- a/golem-component-service-base/src/repo/component.rs +++ b/golem-component-service-base/src/repo/component.rs @@ -201,36 +201,50 @@ impl ComponentRepo for DbComponentRepo { ) -> Result, RepoError> { sqlx::query_as::<_, ComponentRecord>( r#" - SELECT c.namespace AS namespace, c.name AS name, c.component_id AS component_id, - cv.version AS version, cv.size AS size, cv.user_component AS user_component, - cv.protected_component AS protected_component, cv.protector_version AS protector_version, - cv.metadata AS metadata - FROM components c JOIN component_versions cv ON c.component_id = cv.component_id + SELECT + c.namespace AS namespace, + c.name AS name, + c.component_id AS component_id, + cv.version AS version, + cv.size AS size, + cv.user_component AS user_component, + cv.protected_component AS protected_component, + cv.protector_version AS protector_version, + cv.metadata AS metadata + FROM components c + JOIN component_versions cv ON c.component_id = cv.component_id WHERE c.namespace = $1 AND c.component_id = $2 - "# + "#, ) - .bind(namespace) - .bind(component_id) - .fetch_all(self.db_pool.deref()) - .await - .map_err(|e| e.into()) + .bind(namespace) + .bind(component_id) + .fetch_all(self.db_pool.deref()) + .await + .map_err(|e| e.into()) } async fn get_all(&self, namespace: &str) -> Result, RepoError> { sqlx::query_as::<_, ComponentRecord>( r#" - SELECT c.namespace AS namespace, c.name AS name, c.component_id AS component_id, - cv.version AS version, cv.size AS size, cv.user_component AS user_component, - cv.protected_component AS protected_component, cv.protector_version AS protector_version, - cv.metadata AS metadata - FROM components c JOIN component_versions cv ON c.component_id = cv.component_id + SELECT + c.namespace AS namespace, + c.name AS name, + c.component_id AS component_id, + cv.version AS version, + cv.size AS size, + cv.user_component AS user_component, + cv.protected_component AS protected_component, + cv.protector_version AS protector_version, + cv.metadata AS metadata + FROM components c + JOIN component_versions cv ON c.component_id = cv.component_id WHERE c.namespace = $1 - "# + "#, ) - .bind(namespace) - .fetch_all(self.db_pool.deref()) - .await - .map_err(|e| e.into()) + .bind(namespace) + .fetch_all(self.db_pool.deref()) + .await + .map_err(|e| e.into()) } async fn get_latest_version( @@ -240,20 +254,27 @@ impl ComponentRepo for DbComponentRepo { ) -> Result, RepoError> { sqlx::query_as::<_, ComponentRecord>( r#" - SELECT c.namespace AS namespace, c.name AS name, c.component_id AS component_id, - cv.version AS version, cv.size AS size, cv.user_component AS user_component, - cv.protected_component AS protected_component, cv.protector_version AS protector_version, - cv.metadata AS metadata - FROM components c JOIN component_versions cv ON c.component_id = cv.component_id + SELECT + c.namespace AS namespace, + c.name AS name, + c.component_id AS component_id, + cv.version AS version, + cv.size AS size, + cv.user_component AS user_component, + cv.protected_component AS protected_component, + cv.protector_version AS protector_version, + cv.metadata AS metadata + FROM components c + JOIN component_versions cv ON c.component_id = cv.component_id WHERE c.namespace = $1 AND c.component_id = $2 ORDER BY cv.version DESC LIMIT 1 "#, ) - .bind(namespace) - .bind(component_id) - .fetch_optional(self.db_pool.deref()) - .await - .map_err(|e| e.into()) + .bind(namespace) + .bind(component_id) + .fetch_optional(self.db_pool.deref()) + .await + .map_err(|e| e.into()) } async fn get_by_version( @@ -264,20 +285,27 @@ impl ComponentRepo for DbComponentRepo { ) -> Result, RepoError> { sqlx::query_as::<_, ComponentRecord>( r#" - SELECT c.namespace AS namespace, c.name AS name, c.component_id AS component_id, - cv.version AS version, cv.size AS size, cv.user_component AS user_component, - cv.protected_component AS protected_component, cv.protector_version AS protector_version, - cv.metadata AS metadata - FROM components c JOIN component_versions cv ON c.component_id = cv.component_id + SELECT + c.namespace AS namespace, + c.name AS name, + c.component_id AS component_id, + cv.version AS version, + cv.size AS size, + cv.user_component AS user_component, + cv.protected_component AS protected_component, + cv.protector_version AS protector_version, + cv.metadata AS metadata + FROM components c + JOIN component_versions cv ON c.component_id = cv.component_id WHERE c.namespace = $1 AND c.component_id = $2 AND cv.version = $3 "#, ) - .bind(namespace) - .bind(component_id) - .bind(version as i64) - .fetch_optional(self.db_pool.deref()) - .await - .map_err(|e| e.into()) + .bind(namespace) + .bind(component_id) + .bind(version as i64) + .fetch_optional(self.db_pool.deref()) + .await + .map_err(|e| e.into()) } async fn get_by_name( @@ -287,19 +315,26 @@ impl ComponentRepo for DbComponentRepo { ) -> Result, RepoError> { sqlx::query_as::<_, ComponentRecord>( r#" - SELECT c.namespace AS namespace, c.name AS name, c.component_id AS component_id, - cv.version AS version, cv.size AS size, cv.user_component AS user_component, - cv.protected_component AS protected_component, cv.protector_version AS protector_version, - cv.metadata AS metadata - FROM components c JOIN component_versions cv ON c.component_id = cv.component_id + SELECT + c.namespace AS namespace, + c.name AS name, + c.component_id AS component_id, + cv.version AS version, + cv.size AS size, + cv.user_component AS user_component, + cv.protected_component AS protected_component, + cv.protector_version AS protector_version, + cv.metadata AS metadata + FROM components c + JOIN component_versions cv ON c.component_id = cv.component_id WHERE c.namespace = $1 AND c.name = $2 "#, ) - .bind(namespace) - .bind(name) - .fetch_all(self.db_pool.deref()) - .await - .map_err(|e| e.into()) + .bind(namespace) + .bind(name) + .fetch_all(self.db_pool.deref()) + .await + .map_err(|e| e.into()) } async fn get_id_by_name(&self, namespace: &str, name: &str) -> Result, RepoError> { @@ -324,11 +359,16 @@ impl ComponentRepo for DbComponentRepo { async fn delete(&self, namespace: &str, component_id: &Uuid) -> Result<(), RepoError> { let mut transaction = self.db_pool.begin().await?; - sqlx::query("DELETE FROM component_versions WHERE component_id IN (SELECT component_id FROM components WHERE namespace = $1 AND component_id = $2)") - .bind(namespace) - .bind(component_id) - .execute(&mut *transaction) - .await?; + sqlx::query( + r#" + DELETE FROM component_versions + WHERE component_id IN (SELECT component_id FROM components WHERE namespace = $1 AND component_id = $2) + "# + ) + .bind(namespace) + .bind(component_id) + .execute(&mut *transaction) + .await?; sqlx::query("DELETE FROM components WHERE namespace = $1 AND component_id = $2") .bind(namespace) @@ -402,36 +442,50 @@ impl ComponentRepo for DbComponentRepo { ) -> Result, RepoError> { sqlx::query_as::<_, ComponentRecord>( r#" - SELECT c.namespace AS namespace, c.name AS name, c.component_id AS component_id, - cv.version AS version, cv.size AS size, cv.user_component AS user_component, - cv.protected_component AS protected_component, cv.protector_version AS protector_version, - cv.metadata AS metadata - FROM components c JOIN component_versions cv ON c.component_id = cv.component_id + SELECT + c.namespace AS namespace, + c.name AS name, + c.component_id AS component_id, + cv.version AS version, + cv.size AS size, + cv.user_component AS user_component, + cv.protected_component AS protected_component, + cv.protector_version AS protector_version, + cv.metadata AS metadata + FROM components c + JOIN component_versions cv ON c.component_id = cv.component_id WHERE c.namespace = $1 AND c.component_id = $2 - "# + "#, ) - .bind(namespace) - .bind(component_id) - .fetch_all(self.db_pool.deref()) - .await - .map_err(|e| e.into()) + .bind(namespace) + .bind(component_id) + .fetch_all(self.db_pool.deref()) + .await + .map_err(|e| e.into()) } async fn get_all(&self, namespace: &str) -> Result, RepoError> { sqlx::query_as::<_, ComponentRecord>( r#" - SELECT c.namespace AS namespace, c.name AS name, c.component_id AS component_id, - cv.version AS version, cv.size AS size, cv.user_component AS user_component, - cv.protected_component AS protected_component, cv.protector_version AS protector_version, - cv.metadata AS metadata - FROM components c JOIN component_versions cv ON c.component_id = cv.component_id + SELECT + c.namespace AS namespace, + c.name AS name, + c.component_id AS component_id, + cv.version AS version, + cv.size AS size, + cv.user_component AS user_component, + cv.protected_component AS protected_component, + cv.protector_version AS protector_version, + cv.metadata AS metadata + FROM components c + JOIN component_versions cv ON c.component_id = cv.component_id WHERE c.namespace = $1 - "# + "#, ) - .bind(namespace) - .fetch_all(self.db_pool.deref()) - .await - .map_err(|e| e.into()) + .bind(namespace) + .fetch_all(self.db_pool.deref()) + .await + .map_err(|e| e.into()) } async fn get_latest_version( @@ -441,20 +495,27 @@ impl ComponentRepo for DbComponentRepo { ) -> Result, RepoError> { sqlx::query_as::<_, ComponentRecord>( r#" - SELECT c.namespace AS namespace, c.name AS name, c.component_id AS component_id, - cv.version AS version, cv.size AS size, cv.user_component AS user_component, - cv.protected_component AS protected_component, cv.protector_version AS protector_version, - cv.metadata AS metadata - FROM components c JOIN component_versions cv ON c.component_id = cv.component_id + SELECT + c.namespace AS namespace, + c.name AS name, + c.component_id AS component_id, + cv.version AS version, + cv.size AS size, + cv.user_component AS user_component, + cv.protected_component AS protected_component, + cv.protector_version AS protector_version, + cv.metadata AS metadata + FROM components c + JOIN component_versions cv ON c.component_id = cv.component_id WHERE c.namespace = $1 AND c.component_id = $2 ORDER BY cv.version DESC LIMIT 1 "#, ) - .bind(namespace) - .bind(component_id) - .fetch_optional(self.db_pool.deref()) - .await - .map_err(|e| e.into()) + .bind(namespace) + .bind(component_id) + .fetch_optional(self.db_pool.deref()) + .await + .map_err(|e| e.into()) } async fn get_by_version( @@ -465,20 +526,27 @@ impl ComponentRepo for DbComponentRepo { ) -> Result, RepoError> { sqlx::query_as::<_, ComponentRecord>( r#" - SELECT c.namespace AS namespace, c.name AS name, c.component_id AS component_id, - cv.version AS version, cv.size AS size, cv.user_component AS user_component, - cv.protected_component AS protected_component, cv.protector_version AS protector_version, - cv.metadata AS metadata - FROM components c JOIN component_versions cv ON c.component_id = cv.component_id + SELECT + c.namespace AS namespace, + c.name AS name, + c.component_id AS component_id, + cv.version AS version, + cv.size AS size, + cv.user_component AS user_component, + cv.protected_component AS protected_component, + cv.protector_version AS protector_version, + cv.metadata AS metadata + FROM components c + JOIN component_versions cv ON c.component_id = cv.component_id WHERE c.namespace = $1 AND c.component_id = $2 AND cv.version = $3 "#, ) - .bind(namespace) - .bind(component_id) - .bind(version as i64) - .fetch_optional(self.db_pool.deref()) - .await - .map_err(|e| e.into()) + .bind(namespace) + .bind(component_id) + .bind(version as i64) + .fetch_optional(self.db_pool.deref()) + .await + .map_err(|e| e.into()) } async fn get_by_name( @@ -488,19 +556,26 @@ impl ComponentRepo for DbComponentRepo { ) -> Result, RepoError> { sqlx::query_as::<_, ComponentRecord>( r#" - SELECT c.namespace AS namespace, c.name AS name, c.component_id AS component_id, - cv.version AS version, cv.size AS size, cv.user_component AS user_component, - cv.protected_component AS protected_component, cv.protector_version AS protector_version, - cv.metadata AS metadata - FROM components c JOIN component_versions cv ON c.component_id = cv.component_id + SELECT + c.namespace AS namespace, + c.name AS name, + c.component_id AS component_id, + cv.version AS version, + cv.size AS size, + cv.user_component AS user_component, + cv.protected_component AS protected_component, + cv.protector_version AS protector_version, + cv.metadata AS metadata + FROM components c + JOIN component_versions cv ON c.component_id = cv.component_id WHERE c.namespace = $1 AND c.name = $2 "#, ) - .bind(namespace) - .bind(name) - .fetch_all(self.db_pool.deref()) - .await - .map_err(|e| e.into()) + .bind(namespace) + .bind(name) + .fetch_all(self.db_pool.deref()) + .await + .map_err(|e| e.into()) } async fn get_id_by_name(&self, namespace: &str, name: &str) -> Result, RepoError> { @@ -525,11 +600,16 @@ impl ComponentRepo for DbComponentRepo { async fn delete(&self, namespace: &str, component_id: &Uuid) -> Result<(), RepoError> { let mut transaction = self.db_pool.begin().await?; - sqlx::query("DELETE FROM component_versions WHERE component_id IN (SELECT component_id FROM components WHERE namespace = $1 AND component_id = $2)") - .bind(namespace) - .bind(component_id) - .execute(&mut *transaction) - .await?; + sqlx::query( + r#" + DELETE FROM component_versions + WHERE component_id IN (SELECT component_id FROM components WHERE namespace = $1 AND component_id = $2) + "# + ) + .bind(namespace) + .bind(component_id) + .execute(&mut *transaction) + .await?; sqlx::query("DELETE FROM components WHERE namespace = $1 AND component_id = $2") .bind(namespace) diff --git a/golem-component-service-base/src/service/component.rs b/golem-component-service-base/src/service/component.rs index 429c1087c1..2ed1eb81ca 100644 --- a/golem-component-service-base/src/service/component.rs +++ b/golem-component-service-base/src/service/component.rs @@ -58,6 +58,35 @@ impl From for ComponentError { } } +pub fn create_new_component( + component_id: &ComponentId, + component_name: &ComponentName, + data: &[u8], +) -> Result { + let metadata = process_component(data)?; + + let versioned_component_id = VersionedComponentId { + component_id: component_id.clone(), + version: 0, + }; + + let user_component_id = UserComponentId { + versioned_component_id: versioned_component_id.clone(), + }; + let protected_component_id = ProtectedComponentId { + versioned_component_id: versioned_component_id.clone(), + }; + + Ok(Component { + component_name: component_name.clone(), + component_size: data.len() as u64, + metadata, + versioned_component_id, + user_component_id, + protected_component_id, + }) +} + #[async_trait] pub trait ComponentService { async fn create( @@ -176,51 +205,24 @@ where .await? .map_or(Ok(()), |id| Err(ComponentError::AlreadyExists(id)))?; - let metadata = process_component(&data)?; - - let versioned_component_id = VersionedComponentId { - component_id: component_id.clone(), - version: 0, - }; - - let user_component_id = UserComponentId { - versioned_component_id: versioned_component_id.clone(), - }; - let protected_component_id = ProtectedComponentId { - versioned_component_id: versioned_component_id.clone(), - }; + let component = create_new_component(component_id, component_name, &data)?; info!( "Uploaded component - namespace: {}, id: {}, version: 0, exports {:?}", - namespace, versioned_component_id.component_id, metadata.exports + namespace, component_id, component.metadata.exports ); - - let component_size: u64 = data - .len() - .try_into() - .map_err(|e| ComponentError::internal(e, "Failed to convert data length"))?; - tokio::try_join!( - self.upload_user_component(&user_component_id, data.clone()), - self.upload_protected_component(&protected_component_id, data) + self.upload_user_component(&component.user_component_id, data.clone()), + self.upload_protected_component(&component.protected_component_id, data) )?; - let component = Component { - component_name: component_name.clone(), - component_size, - metadata, - versioned_component_id, - user_component_id, - protected_component_id, - }; - let record = ComponentRecord::new(namespace, component.clone()) .map_err(|e| ComponentError::internal(e, "Failed to convert record"))?; self.component_repo.create(&record).await?; self.component_compilation - .enqueue_compilation(&component.versioned_component_id.component_id, 0) + .enqueue_compilation(component_id, component.versioned_component_id.version) .await; Ok(component) diff --git a/golem-component-service-base/tests/services_tests.rs b/golem-component-service-base/tests/services_tests.rs index 230c3999df..fadb813032 100644 --- a/golem-component-service-base/tests/services_tests.rs +++ b/golem-component-service-base/tests/services_tests.rs @@ -5,9 +5,11 @@ mod tests { use golem_service_base::db; use golem_common::model::ComponentId; - use golem_component_service_base::repo::component::{ComponentRepo, DbComponentRepo}; + use golem_component_service_base::repo::component::{ + ComponentRecord, ComponentRepo, DbComponentRepo, + }; use golem_component_service_base::service::component::{ - ComponentService, ComponentServiceDefault, + create_new_component, ComponentService, ComponentServiceDefault, }; use golem_component_service_base::service::component_compilation::{ ComponentCompilationService, ComponentCompilationServiceDisabled, @@ -18,6 +20,7 @@ mod tests { use testcontainers::clients::Cli; use testcontainers::{Container, RunnableImage}; use testcontainers_modules::postgres::Postgres; + use uuid::Uuid; fn start_docker_postgres<'d>(docker: &'d Cli) -> (DbPostgresConfig, Container<'d, Postgres>) { let image = RunnableImage::from(Postgres::default()).with_tag("14.7-alpine"); @@ -68,7 +71,8 @@ mod tests { let component_repo: Arc = Arc::new(DbComponentRepo::new(db_pool.clone().into())); - test_services(component_repo).await; + test_repo(component_repo.clone()).await; + test_services(component_repo.clone()).await; } #[tokio::test] @@ -88,7 +92,13 @@ mod tests { let component_repo: Arc = Arc::new(DbComponentRepo::new(db_pool.clone().into())); - test_services(component_repo).await; + test_repo(component_repo.clone()).await; + test_services(component_repo.clone()).await; + } + + fn get_component_data(name: &str) -> Vec { + let path = format!("../test-components/{}.wasm", name); + std::fs::read(path).unwrap() } async fn test_services(component_repo: Arc) { @@ -111,11 +121,6 @@ mod tests { compilation_service.clone(), )); - fn get_component_data(name: &str) -> Vec { - let path = format!("../test-components/{}.wasm", name); - std::fs::read(path).unwrap() - } - let component_name1 = ComponentName("shopping-cart".to_string()); let component_name2 = ComponentName("rust-echo".to_string()); @@ -255,4 +260,109 @@ mod tests { .unwrap(); assert!(component_result.len() == 3); } + + async fn test_repo(component_repo: Arc) { + test_repo_component_id_unique(component_repo.clone()).await; + test_repo_component_name_unique_in_namespace(component_repo.clone()).await; + test_repo_component_delete(component_repo.clone()).await; + } + + async fn test_repo_component_id_unique(component_repo: Arc) { + let namespace1 = Uuid::new_v4().to_string(); + let namespace2 = Uuid::new_v4().to_string(); + + let component_name1 = ComponentName("shopping-cart1".to_string()); + let data = get_component_data("shopping-cart"); + + let component1 = + create_new_component(&ComponentId::new_v4(), &component_name1, &data).unwrap(); + + let result1 = component_repo + .create(&ComponentRecord::new(namespace1.clone(), component1.clone()).unwrap()) + .await; + let result2 = component_repo + .create( + &ComponentRecord::new(namespace1.clone(), component1.clone().next_version()) + .unwrap(), + ) + .await; + let result3 = component_repo + .create(&ComponentRecord::new(namespace2.clone(), component1.clone()).unwrap()) + .await; + + assert!(result1.is_ok()); + assert!(result2.is_ok()); + assert!(result3.is_err()); + } + + async fn test_repo_component_name_unique_in_namespace( + component_repo: Arc, + ) { + let namespace1 = Uuid::new_v4().to_string(); + let namespace2 = Uuid::new_v4().to_string(); + + let component_name1 = ComponentName("shopping-cart1".to_string()); + let data = get_component_data("shopping-cart"); + + let component1 = + create_new_component(&ComponentId::new_v4(), &component_name1, &data).unwrap(); + let component2 = + create_new_component(&ComponentId::new_v4(), &component_name1, &data).unwrap(); + + let result1 = component_repo + .create(&ComponentRecord::new(namespace1.clone(), component1.clone()).unwrap()) + .await; + let result2 = component_repo + .create(&ComponentRecord::new(namespace1.clone(), component2.clone()).unwrap()) + .await; + let result3 = component_repo + .create(&ComponentRecord::new(namespace2.clone(), component2.clone()).unwrap()) + .await; + + assert!(result1.is_ok()); + assert!(result2.is_err()); + assert!(result3.is_ok()); + } + + async fn test_repo_component_delete(component_repo: Arc) { + let namespace1 = Uuid::new_v4().to_string(); + + let component_name1 = ComponentName("shopping-cart1".to_string()); + let data = get_component_data("shopping-cart"); + + let component1 = + create_new_component(&ComponentId::new_v4(), &component_name1, &data).unwrap(); + + let result1 = component_repo + .create(&ComponentRecord::new(namespace1.clone(), component1.clone()).unwrap()) + .await; + + let result2 = component_repo + .get( + &namespace1, + &component1.versioned_component_id.component_id.0, + ) + .await; + + let result3 = component_repo + .delete( + &namespace1, + &component1.versioned_component_id.component_id.0, + ) + .await; + + let result4 = component_repo + .get( + &namespace1, + &component1.versioned_component_id.component_id.0, + ) + .await; + + assert!(result1.is_ok()); + assert!(result2.is_ok()); + assert_eq!(result2.unwrap().len(), 1); + assert!(result3.is_ok()); + assert!(result4.is_ok()); + assert!(result4.unwrap().is_empty()); + } } From 497df2d99a3a81b1690b25f9bcc4780a311dfae2 Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Wed, 3 Jul 2024 18:34:20 +0200 Subject: [PATCH 18/30] remove unnecessary fields from component_versions table --- .../src/repo/component.rs | 96 ++++++------------- .../tests/db/migration/postgres/001__init.sql | 3 - .../tests/db/migration/sqlite/001__init.sql | 3 - .../db/migration/postgres/001__init.sql | 3 - .../db/migration/sqlite/001__init.sql | 3 - .../src/repo/api_definition.rs | 14 +-- 6 files changed, 38 insertions(+), 84 deletions(-) diff --git a/golem-component-service-base/src/repo/component.rs b/golem-component-service-base/src/repo/component.rs index f208541e48..198e42a4d5 100644 --- a/golem-component-service-base/src/repo/component.rs +++ b/golem-component-service-base/src/repo/component.rs @@ -32,9 +32,6 @@ pub struct ComponentRecord { pub name: String, pub size: i32, pub version: i64, - pub user_component: String, - pub protected_component: String, - pub protector_version: Option, pub metadata: Vec, } @@ -84,9 +81,6 @@ impl ComponentRecord { name: component.component_name.0, size: component.component_size as i32, version: component.versioned_component_id.version as i64, - user_component: component.versioned_component_id.slug(), - protected_component: component.protected_component_id.slug(), - protector_version: None, metadata: metadata.into(), }) } @@ -153,7 +147,9 @@ impl ComponentRepo for DbComponentRepo { if let Some(result) = result { let namespace: String = result.get("namespace"); if namespace != component.namespace { - return Err(RepoError::Internal("Component id not unique".to_string())); + return Err(RepoError::Internal( + "Component namespace invalid".to_string(), + )); } } else { sqlx::query( @@ -174,20 +170,17 @@ impl ComponentRepo for DbComponentRepo { sqlx::query( r#" INSERT INTO component_versions - (component_id, version, size, user_component, protected_component, protector_version, metadata) + (component_id, version, size, metadata) VALUES - ($1, $2, $3, $4, $5, $6, $7) + ($1, $2, $3, $4) "#, ) - .bind(component.component_id) - .bind(component.version) - .bind(component.size) - .bind(component.user_component.clone()) - .bind(component.protected_component.clone()) - .bind(component.protector_version) - .bind(component.metadata.clone()) - .execute(&mut *transaction) - .await?; + .bind(component.component_id) + .bind(component.version) + .bind(component.size) + .bind(component.metadata.clone()) + .execute(&mut *transaction) + .await?; transaction.commit().await?; @@ -207,9 +200,6 @@ impl ComponentRepo for DbComponentRepo { c.component_id AS component_id, cv.version AS version, cv.size AS size, - cv.user_component AS user_component, - cv.protected_component AS protected_component, - cv.protector_version AS protector_version, cv.metadata AS metadata FROM components c JOIN component_versions cv ON c.component_id = cv.component_id @@ -232,9 +222,6 @@ impl ComponentRepo for DbComponentRepo { c.component_id AS component_id, cv.version AS version, cv.size AS size, - cv.user_component AS user_component, - cv.protected_component AS protected_component, - cv.protector_version AS protector_version, cv.metadata AS metadata FROM components c JOIN component_versions cv ON c.component_id = cv.component_id @@ -260,9 +247,6 @@ impl ComponentRepo for DbComponentRepo { c.component_id AS component_id, cv.version AS version, cv.size AS size, - cv.user_component AS user_component, - cv.protected_component AS protected_component, - cv.protector_version AS protector_version, cv.metadata AS metadata FROM components c JOIN component_versions cv ON c.component_id = cv.component_id @@ -291,9 +275,6 @@ impl ComponentRepo for DbComponentRepo { c.component_id AS component_id, cv.version AS version, cv.size AS size, - cv.user_component AS user_component, - cv.protected_component AS protected_component, - cv.protector_version AS protector_version, cv.metadata AS metadata FROM components c JOIN component_versions cv ON c.component_id = cv.component_id @@ -321,9 +302,6 @@ impl ComponentRepo for DbComponentRepo { c.component_id AS component_id, cv.version AS version, cv.size AS size, - cv.user_component AS user_component, - cv.protected_component AS protected_component, - cv.protector_version AS protector_version, cv.metadata AS metadata FROM components c JOIN component_versions cv ON c.component_id = cv.component_id @@ -394,7 +372,9 @@ impl ComponentRepo for DbComponentRepo { if let Some(result) = result { let namespace: String = result.get("namespace"); if namespace != component.namespace { - return Err(RepoError::Internal("Component id not unique".to_string())); + return Err(RepoError::Internal( + "Component namespace invalid".to_string(), + )); } } else { sqlx::query( @@ -415,20 +395,17 @@ impl ComponentRepo for DbComponentRepo { sqlx::query( r#" INSERT INTO component_versions - (component_id, version, size, user_component, protected_component, protector_version, metadata) + (component_id, version, size, metadata) VALUES - ($1, $2, $3, $4, $5, $6, $7) + ($1, $2, $3, $4) "#, ) - .bind(component.component_id) - .bind(component.version) - .bind(component.size) - .bind(component.user_component.clone()) - .bind(component.protected_component.clone()) - .bind(component.protector_version) - .bind(component.metadata.clone()) - .execute(&mut *transaction) - .await?; + .bind(component.component_id) + .bind(component.version) + .bind(component.size) + .bind(component.metadata.clone()) + .execute(&mut *transaction) + .await?; transaction.commit().await?; @@ -448,9 +425,6 @@ impl ComponentRepo for DbComponentRepo { c.component_id AS component_id, cv.version AS version, cv.size AS size, - cv.user_component AS user_component, - cv.protected_component AS protected_component, - cv.protector_version AS protector_version, cv.metadata AS metadata FROM components c JOIN component_versions cv ON c.component_id = cv.component_id @@ -473,9 +447,6 @@ impl ComponentRepo for DbComponentRepo { c.component_id AS component_id, cv.version AS version, cv.size AS size, - cv.user_component AS user_component, - cv.protected_component AS protected_component, - cv.protector_version AS protector_version, cv.metadata AS metadata FROM components c JOIN component_versions cv ON c.component_id = cv.component_id @@ -501,9 +472,6 @@ impl ComponentRepo for DbComponentRepo { c.component_id AS component_id, cv.version AS version, cv.size AS size, - cv.user_component AS user_component, - cv.protected_component AS protected_component, - cv.protector_version AS protector_version, cv.metadata AS metadata FROM components c JOIN component_versions cv ON c.component_id = cv.component_id @@ -532,9 +500,6 @@ impl ComponentRepo for DbComponentRepo { c.component_id AS component_id, cv.version AS version, cv.size AS size, - cv.user_component AS user_component, - cv.protected_component AS protected_component, - cv.protector_version AS protector_version, cv.metadata AS metadata FROM components c JOIN component_versions cv ON c.component_id = cv.component_id @@ -562,9 +527,6 @@ impl ComponentRepo for DbComponentRepo { c.component_id AS component_id, cv.version AS version, cv.size AS size, - cv.user_component AS user_component, - cv.protected_component AS protected_component, - cv.protector_version AS protector_version, cv.metadata AS metadata FROM components c JOIN component_versions cv ON c.component_id = cv.component_id @@ -623,24 +585,26 @@ impl ComponentRepo for DbComponentRepo { } pub mod record_metadata_serde { - use bincode::{Decode, Encode}; use bytes::Bytes; use golem_common::serialization::serialize_with_version; + use golem_service_base::model::ComponentMetadata; + pub const SERIALIZATION_VERSION_V1: u8 = 1u8; - pub fn serialize(routes: &T) -> Result { - serialize_with_version(routes, SERIALIZATION_VERSION_V1) + pub fn serialize(value: &ComponentMetadata) -> Result { + // TODO use golem_api_grpc::proto::golem::component::ComponentMetadata + serialize_with_version(value, SERIALIZATION_VERSION_V1) } - pub fn deserialize(bytes: &[u8]) -> Result { + pub fn deserialize(bytes: &[u8]) -> Result { let (version, data) = bytes.split_at(1); match version[0] { SERIALIZATION_VERSION_V1 => { - let (routes, _) = bincode::decode_from_slice(data, bincode::config::standard()) + let (value, _) = bincode::decode_from_slice(data, bincode::config::standard()) .map_err(|e| format!("Failed to deserialize value: {e}"))?; - Ok(routes) + Ok(value) } _ => Err("Unsupported serialization version".to_string()), } diff --git a/golem-component-service-base/tests/db/migration/postgres/001__init.sql b/golem-component-service-base/tests/db/migration/postgres/001__init.sql index 5c8f7dea1c..66d60275e1 100644 --- a/golem-component-service-base/tests/db/migration/postgres/001__init.sql +++ b/golem-component-service-base/tests/db/migration/postgres/001__init.sql @@ -15,9 +15,6 @@ CREATE TABLE component_versions version bigint NOT NULL, size integer NOT NULL, created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - user_component text NOT NULL, - protected_component text NOT NULL, - protector_version bigint, metadata bytea NOT NULL, PRIMARY KEY (component_id, version) ); diff --git a/golem-component-service-base/tests/db/migration/sqlite/001__init.sql b/golem-component-service-base/tests/db/migration/sqlite/001__init.sql index 0bb14b9dcb..fa088d860e 100644 --- a/golem-component-service-base/tests/db/migration/sqlite/001__init.sql +++ b/golem-component-service-base/tests/db/migration/sqlite/001__init.sql @@ -15,9 +15,6 @@ CREATE TABLE component_versions version bigint NOT NULL, size integer NOT NULL, created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, - user_component text NOT NULL, - protected_component text NOT NULL, - protector_version bigint, metadata blob NOT NULL, PRIMARY KEY (component_id, version) ); diff --git a/golem-component-service/db/migration/postgres/001__init.sql b/golem-component-service/db/migration/postgres/001__init.sql index 5c8f7dea1c..66d60275e1 100644 --- a/golem-component-service/db/migration/postgres/001__init.sql +++ b/golem-component-service/db/migration/postgres/001__init.sql @@ -15,9 +15,6 @@ CREATE TABLE component_versions version bigint NOT NULL, size integer NOT NULL, created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - user_component text NOT NULL, - protected_component text NOT NULL, - protector_version bigint, metadata bytea NOT NULL, PRIMARY KEY (component_id, version) ); diff --git a/golem-component-service/db/migration/sqlite/001__init.sql b/golem-component-service/db/migration/sqlite/001__init.sql index 0bb14b9dcb..fa088d860e 100644 --- a/golem-component-service/db/migration/sqlite/001__init.sql +++ b/golem-component-service/db/migration/sqlite/001__init.sql @@ -15,9 +15,6 @@ CREATE TABLE component_versions version bigint NOT NULL, size integer NOT NULL, created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, - user_component text NOT NULL, - protected_component text NOT NULL, - protector_version bigint, metadata blob NOT NULL, PRIMARY KEY (component_id, version) ); diff --git a/golem-worker-service-base/src/repo/api_definition.rs b/golem-worker-service-base/src/repo/api_definition.rs index cbd31d261d..f598a6e97d 100644 --- a/golem-worker-service-base/src/repo/api_definition.rs +++ b/golem-worker-service-base/src/repo/api_definition.rs @@ -487,24 +487,26 @@ impl ApiDefinitionRepo for InMemoryApiDefinitionRepo { } pub mod record_data_serde { - use bincode::{Decode, Encode}; + use crate::api_definition::http::Route; use bytes::Bytes; use golem_common::serialization::serialize_with_version; + pub const SERIALIZATION_VERSION_V1: u8 = 1u8; - pub fn serialize(routes: &T) -> Result { - serialize_with_version(routes, SERIALIZATION_VERSION_V1) + pub fn serialize(value: &Vec) -> Result { + // TODO use golem_api_grpc::proto::golem::api_definition::http::Route + serialize_with_version(value, SERIALIZATION_VERSION_V1) } - pub fn deserialize(bytes: &[u8]) -> Result { + pub fn deserialize(bytes: &[u8]) -> Result, String> { let (version, data) = bytes.split_at(1); match version[0] { SERIALIZATION_VERSION_V1 => { - let (routes, _) = bincode::decode_from_slice(data, bincode::config::standard()) + let (value, _) = bincode::decode_from_slice(data, bincode::config::standard()) .map_err(|e| format!("Failed to deserialize value: {e}"))?; - Ok(routes) + Ok(value) } _ => Err("Unsupported serialization version".to_string()), } From da66f3927cdca1b723aa3762117e7160fe2efb3e Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Wed, 3 Jul 2024 20:11:27 +0200 Subject: [PATCH 19/30] components and api_definition - proto serde --- Cargo.lock | 2 ++ golem-component-service-base/Cargo.toml | 1 + .../src/repo/component.rs | 16 ++++++---- golem-worker-service-base/Cargo.toml | 1 + .../src/repo/api_definition.rs | 29 +++++++++++++++---- 5 files changed, 37 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f1458531f6..4263bb17e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3534,6 +3534,7 @@ dependencies = [ "golem-service-base", "golem-wasm-ast", "http 0.2.12", + "prost", "serde 1.0.203", "sqlx", "thiserror", @@ -3981,6 +3982,7 @@ dependencies = [ "poem", "poem-openapi", "prometheus", + "prost", "regex", "rustc-hash", "serde 1.0.203", diff --git a/golem-component-service-base/Cargo.toml b/golem-component-service-base/Cargo.toml index 4ff276e192..46184f7e01 100644 --- a/golem-component-service-base/Cargo.toml +++ b/golem-component-service-base/Cargo.toml @@ -16,6 +16,7 @@ async-trait = { workspace = true } bincode = { workspace = true } bytes = { workspace = true } http_02 = { workspace = true } +prost = { workspace = true } serde = { workspace = true } sqlx = { workspace = true, features = [ "runtime-tokio", diff --git a/golem-component-service-base/src/repo/component.rs b/golem-component-service-base/src/repo/component.rs index 198e42a4d5..d8f994ebce 100644 --- a/golem-component-service-base/src/repo/component.rs +++ b/golem-component-service-base/src/repo/component.rs @@ -585,15 +585,19 @@ impl ComponentRepo for DbComponentRepo { } pub mod record_metadata_serde { - use bytes::Bytes; - use golem_common::serialization::serialize_with_version; + use bytes::{BufMut, Bytes, BytesMut}; + use golem_api_grpc::proto::golem::component::ComponentMetadata as ComponentMetadataProto; use golem_service_base::model::ComponentMetadata; + use prost::Message; pub const SERIALIZATION_VERSION_V1: u8 = 1u8; pub fn serialize(value: &ComponentMetadata) -> Result { - // TODO use golem_api_grpc::proto::golem::component::ComponentMetadata - serialize_with_version(value, SERIALIZATION_VERSION_V1) + let proto_value: ComponentMetadataProto = value.clone().into(); + let mut bytes = BytesMut::new(); + bytes.put_u8(SERIALIZATION_VERSION_V1); + bytes.extend_from_slice(&proto_value.encode_to_vec()); + Ok(bytes.freeze()) } pub fn deserialize(bytes: &[u8]) -> Result { @@ -601,9 +605,9 @@ pub mod record_metadata_serde { match version[0] { SERIALIZATION_VERSION_V1 => { - let (value, _) = bincode::decode_from_slice(data, bincode::config::standard()) + let proto_value: ComponentMetadataProto = Message::decode(data) .map_err(|e| format!("Failed to deserialize value: {e}"))?; - + let value = proto_value.try_into()?; Ok(value) } _ => Err("Unsupported serialization version".to_string()), diff --git a/golem-worker-service-base/Cargo.toml b/golem-worker-service-base/Cargo.toml index a36bbdc723..19d7fea185 100644 --- a/golem-worker-service-base/Cargo.toml +++ b/golem-worker-service-base/Cargo.toml @@ -38,6 +38,7 @@ opentelemetry_sdk = { workspace = true } poem = { workspace = true } poem-openapi = { workspace = true } prometheus = { workspace = true } +prost = { workspace = true } regex = { workspace = true } rustc-hash = "1.1.0" serde = { workspace = true } diff --git a/golem-worker-service-base/src/repo/api_definition.rs b/golem-worker-service-base/src/repo/api_definition.rs index f598a6e97d..1647d6c051 100644 --- a/golem-worker-service-base/src/repo/api_definition.rs +++ b/golem-worker-service-base/src/repo/api_definition.rs @@ -488,14 +488,25 @@ impl ApiDefinitionRepo for InMemoryApiDefinitionRepo { pub mod record_data_serde { use crate::api_definition::http::Route; - use bytes::Bytes; - use golem_common::serialization::serialize_with_version; + use bytes::{BufMut, Bytes, BytesMut}; + use golem_api_grpc::proto::golem::apidefinition::{HttpApiDefinition, HttpRoute}; + use prost::Message; pub const SERIALIZATION_VERSION_V1: u8 = 1u8; - pub fn serialize(value: &Vec) -> Result { - // TODO use golem_api_grpc::proto::golem::api_definition::http::Route - serialize_with_version(value, SERIALIZATION_VERSION_V1) + pub fn serialize(value: &[Route]) -> Result { + let routes: Vec = value + .iter() + .cloned() + .map(HttpRoute::try_from) + .collect::, String>>()?; + + let proto_value: HttpApiDefinition = HttpApiDefinition { routes }; + + let mut bytes = BytesMut::new(); + bytes.put_u8(SERIALIZATION_VERSION_V1); + bytes.extend_from_slice(&proto_value.encode_to_vec()); + Ok(bytes.freeze()) } pub fn deserialize(bytes: &[u8]) -> Result, String> { @@ -503,9 +514,15 @@ pub mod record_data_serde { match version[0] { SERIALIZATION_VERSION_V1 => { - let (value, _) = bincode::decode_from_slice(data, bincode::config::standard()) + let proto_value: HttpApiDefinition = Message::decode(data) .map_err(|e| format!("Failed to deserialize value: {e}"))?; + let value = proto_value + .routes + .into_iter() + .map(Route::try_from) + .collect::, String>>()?; + Ok(value) } _ => Err("Unsupported serialization version".to_string()), From c0e8a1e4874dc7022fbb992a1ca5947f21407474 Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Fri, 5 Jul 2024 09:40:49 +0200 Subject: [PATCH 20/30] merge fixes --- Cargo.lock | 9 +++++++++ golem-service-base/src/lib.rs | 6 +++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4263bb17e1..ad41bb630d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3526,9 +3526,12 @@ dependencies = [ name = "golem-component-service-base" version = "0.0.0" dependencies = [ + "anyhow", "async-trait", "bincode", "bytes 1.6.0", + "criterion", + "fastrand 2.1.0", "golem-api-grpc", "golem-common", "golem-service-base", @@ -3537,7 +3540,13 @@ dependencies = [ "prost", "serde 1.0.203", "sqlx", + "tap", + "testcontainers", + "testcontainers-modules", "thiserror", + "tokio", + "tokio-stream", + "tokio-util", "tonic", "tracing", "uuid", diff --git a/golem-service-base/src/lib.rs b/golem-service-base/src/lib.rs index 29dde3dbef..3da6b0c985 100644 --- a/golem-service-base/src/lib.rs +++ b/golem-service-base/src/lib.rs @@ -11,13 +11,13 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - pub mod api_tags; +pub mod auth; pub mod config; +pub mod db; pub mod model; pub mod routing_table; pub mod service; -pub mod typechecker; -pub mod db; pub mod stream; pub mod type_inference; +pub mod typechecker; From 3d3f8f9d1bcbf2fdaf69b0c9b171782287efb966 Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Fri, 5 Jul 2024 10:03:40 +0200 Subject: [PATCH 21/30] WorkerBinding proto - worker_id -> worker_name --- .../proto/golem/apidefinition/api_definition.proto | 2 +- .../src/api/register_api_definition_api.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/golem-api-grpc/proto/golem/apidefinition/api_definition.proto b/golem-api-grpc/proto/golem/apidefinition/api_definition.proto index 1e995bfcb6..f0f99166df 100644 --- a/golem-api-grpc/proto/golem/apidefinition/api_definition.proto +++ b/golem-api-grpc/proto/golem/apidefinition/api_definition.proto @@ -45,7 +45,7 @@ enum HttpMethod { message WorkerBinding { golem.component.ComponentId component = 1; - string worker_id = 2; + string worker_name = 2; string response = 3; optional string idempotency_key = 4; } \ No newline at end of file diff --git a/golem-worker-service-base/src/api/register_api_definition_api.rs b/golem-worker-service-base/src/api/register_api_definition_api.rs index 73e3dee271..09362b5454 100644 --- a/golem-worker-service-base/src/api/register_api_definition_api.rs +++ b/golem-worker-service-base/src/api/register_api_definition_api.rs @@ -311,7 +311,7 @@ impl TryFrom for grpc_apidefinition:: fn try_from(value: crate::worker_binding::GolemWorkerBinding) -> Result { let response: String = value.response.0.to_string(); - let worker_id = rib::to_string(&value.worker_name).map_err(|e| e.to_string())?; + let worker_name = rib::to_string(&value.worker_name).map_err(|e| e.to_string())?; let idempotency_key = if let Some(key) = &value.idempotency_key { Some(rib::to_string(key).map_err(|e| e.to_string())?) @@ -321,7 +321,7 @@ impl TryFrom for grpc_apidefinition:: let result = grpc_apidefinition::WorkerBinding { component: Some(value.component_id.into()), - worker_id, + worker_name, idempotency_key, response, }; @@ -339,7 +339,7 @@ impl TryFrom for crate::worker_binding::Golem crate::worker_binding::ResponseMapping(r) }; - let worker_name = value.worker_id.parse()?; + let worker_name = value.worker_name.parse()?; let component_id = value.component.ok_or("component is missing")?.try_into()?; From 6665f8e3bf6aba141ec573f71e7c8552512ae435 Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Fri, 5 Jul 2024 21:22:36 +0200 Subject: [PATCH 22/30] golem-rib proto --- Cargo.lock | 1 + golem-api-grpc/build.rs | 2 + golem-api-grpc/proto/golem/rib/expr.proto | 186 +++++++ .../proto/golem/rib/function_name.proto | 97 ++++ golem-rib/Cargo.toml | 2 + golem-rib/src/expr.rs | 479 ++++++++++++++++++ golem-rib/src/function_name.rs | 351 +++++++++++-- 7 files changed, 1083 insertions(+), 35 deletions(-) create mode 100644 golem-api-grpc/proto/golem/rib/expr.proto create mode 100644 golem-api-grpc/proto/golem/rib/function_name.proto diff --git a/Cargo.lock b/Cargo.lock index ad41bb630d..3506a1163e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3597,6 +3597,7 @@ dependencies = [ "async-trait", "bincode", "combine", + "golem-api-grpc", "golem-wasm-ast", "golem-wasm-rpc", "semver", diff --git a/golem-api-grpc/build.rs b/golem-api-grpc/build.rs index 6981e3b6f0..7a6ced3df5 100644 --- a/golem-api-grpc/build.rs +++ b/golem-api-grpc/build.rs @@ -37,6 +37,8 @@ fn main() -> Result<(), Box> { .include_file("mod.rs") .compile( &[ + "proto/golem/rib/function_name.proto", + "proto/golem/rib/expr.proto", "proto/golem/common/account_id.proto", "proto/golem/common/project_id.proto", "proto/golem/common/empty.proto", diff --git a/golem-api-grpc/proto/golem/rib/expr.proto b/golem-api-grpc/proto/golem/rib/expr.proto new file mode 100644 index 0000000000..77c104379b --- /dev/null +++ b/golem-api-grpc/proto/golem/rib/expr.proto @@ -0,0 +1,186 @@ +syntax = "proto3"; + +package golem.rib; + +import "golem/rib/function_name.proto"; + +message Expr { + oneof expr { + LetExpr let = 1; + SelectFieldExpr select_field = 2; + SelectIndexExpr select_index = 3; + SequenceExpr sequence = 4; + RecordExpr record = 5; + TupleExpr tuple = 6; + LiteralExpr literal = 7; + NumberExpr number = 8; + FlagsExpr flags = 9; + IdentifierExpr identifier = 10; + BooleanExpr boolean = 11; + ConcatExpr concat = 12; + MultipleExpr multiple = 13; + NotExpr not = 14; + GreaterThanExpr greater_than = 15; + GreaterThanOrEqualToExpr greater_than_or_equal = 16; + LessThanOrEqualToExpr less_than_or_equal = 17; + EqualToExpr equal_to = 18; + LessThanExpr less_than = 19; + CondExpr cond = 20; + PatternMatchExpr pattern_match = 21; + OptionExpr option = 22; + ResultExpr result = 23; + CallExpr call = 24; + } +} + +message LetExpr { + string name = 1; + Expr expr = 2; +} + +message SelectFieldExpr { + string field = 1; + Expr expr = 2; +} + +message SelectIndexExpr { + uint64 index = 1; + Expr expr = 2; +} + +message SequenceExpr { + repeated Expr exprs = 1; +} + +message RecordExpr { + repeated RecordFieldExpr fields = 1; +} + +message RecordFieldExpr { + string name = 1; + Expr expr = 2; +} + +message TupleExpr { + repeated Expr exprs = 1; +} + +message LiteralExpr { + string value = 1; +} + +message NumberExpr { + oneof number { + uint64 unsigned = 1; + int64 signed = 2; + double float = 3; + } +} + +message FlagsExpr { + repeated string values = 1; +} + +message IdentifierExpr { + string name = 1; +} + +message BooleanExpr { + bool value = 1; +} + +message ConcatExpr { + repeated Expr exprs = 1; +} + +message MultipleExpr { + repeated Expr exprs = 1; +} + +message NotExpr { + Expr expr = 1; +} + +message GreaterThanExpr { + Expr left = 1; + Expr right = 2; +} + +message GreaterThanOrEqualToExpr { + Expr left = 1; + Expr right = 2; +} + +message LessThanOrEqualToExpr { + Expr left = 1; + Expr right = 2; +} + +message EqualToExpr { + Expr left = 1; + Expr right = 2; +} + +message LessThanExpr { + Expr left = 1; + Expr right = 2; +} + +message CondExpr { + Expr left = 1; + Expr cond = 2; + Expr right = 3; +} + +message PatternMatchExpr { + Expr expr = 1; + repeated MatchArm patterns = 2; +} + +message OptionExpr { + optional Expr expr = 1; +} + +message ResultExpr { + oneof result { + Expr ok = 1; + Expr err = 2; + } +} + +message CallExpr { + golem.rib.ParsedFunctionName name = 1; + repeated Expr params = 2; +} + +message MatchArm { + ArmPattern pattern = 1; + Expr expr = 2; +} + +message ArmPattern { + oneof pattern { + WildCardArmPattern wild_card = 1; + AsArmPattern as = 2; + ConstructorArmPattern constructor = 3; + LiteralArmPattern literal = 4; + } +} + +message WildCardArmPattern { + +} + +message AsArmPattern { + string name = 1; + ArmPattern pattern = 2; +} + +message ConstructorArmPattern { + string name = 1; + repeated ArmPattern patterns = 2; +} + +message LiteralArmPattern { + Expr expr = 1; +} diff --git a/golem-api-grpc/proto/golem/rib/function_name.proto b/golem-api-grpc/proto/golem/rib/function_name.proto new file mode 100644 index 0000000000..aa9dc5370e --- /dev/null +++ b/golem-api-grpc/proto/golem/rib/function_name.proto @@ -0,0 +1,97 @@ +syntax = "proto3"; + +package golem.rib; + +message ParsedFunctionName { + ParsedFunctionSite site = 1; + ParsedFunctionReference function = 2; +} + +message ParsedFunctionSite { + oneof site { + GlobalFunctionSite global = 1; + InterfaceFunctionSite interface = 2; + PackageInterfaceFunctionSite package_interface = 3; + } +} + +message GlobalFunctionSite { + +} + +message InterfaceFunctionSite { + string name = 1; +} + +message PackageInterfaceFunctionSite { + string namespace = 1; + string package = 2; + string interface = 3; + SemVersion version = 4; +} + +message SemVersion { + uint64 major = 1; + uint64 minor = 2; + uint64 patch = 3; + string pre = 4; + string build = 5; +} + +message ParsedFunctionReference { + oneof function_reference { + FunctionFunctionReference function = 1; + RawResourceConstructorFunctionReference raw_resource_constructor = 2; + RawResourceDropFunctionReference raw_resource_drop = 3; + RawResourceMethodFunctionReference raw_resource_method = 4; + RawResourceStaticMethodFunctionReference raw_resource_static_method = 5; + IndexedResourceConstructorFunctionReference indexed_resource_constructor = 6; + IndexedResourceMethodFunctionReference indexed_resource_method = 7; + IndexedResourceStaticMethodFunctionReference indexed_resource_static_method = 8; + IndexedResourceDropFunctionReference indexed_resource_drop = 9; + } +} + +message FunctionFunctionReference { + string function = 1; +} + +message RawResourceConstructorFunctionReference { + string resource = 1; +} + +message RawResourceDropFunctionReference { + string resource = 1; +} + +message RawResourceMethodFunctionReference { + string resource = 1; + string method = 2; +} + +message RawResourceStaticMethodFunctionReference { + string resource = 1; + string method = 2; +} + +message IndexedResourceConstructorFunctionReference { + string resource = 1; + repeated string resource_params = 2; +} + +message IndexedResourceMethodFunctionReference { + string resource = 1; + repeated string resource_params = 2; + string method = 3; +} + +message IndexedResourceStaticMethodFunctionReference { + string resource = 1; + repeated string resource_params = 2; + string method = 3; +} + +message IndexedResourceDropFunctionReference { + string resource = 1; + repeated string resource_params = 2; +} \ No newline at end of file diff --git a/golem-rib/Cargo.toml b/golem-rib/Cargo.toml index 850656861d..a0f90549e3 100644 --- a/golem-rib/Cargo.toml +++ b/golem-rib/Cargo.toml @@ -8,6 +8,8 @@ repository = "https://github.com/golemcloud/golem" description = "Parser for Golem's Rib language" [dependencies] +golem-api-grpc = { path = "../golem-api-grpc", version = "0.0.0" } + async-trait = { workspace = true } bincode = { workspace = true } combine = "4.6.7" diff --git a/golem-rib/src/expr.rs b/golem-rib/src/expr.rs index 5cc544a6d1..7459d485f4 100644 --- a/golem-rib/src/expr.rs +++ b/golem-rib/src/expr.rs @@ -149,6 +149,349 @@ impl Expr { } } +impl TryFrom for Expr { + type Error = String; + + fn try_from(value: golem_api_grpc::proto::golem::rib::Expr) -> Result { + let expr = value.expr.ok_or("Missing expr")?; + + let expr = match expr { + golem_api_grpc::proto::golem::rib::expr::Expr::Let(expr) => { + let name = expr.name; + let expr = *expr.expr.ok_or("Missing expr")?; + Expr::Let(name, Box::new(expr.try_into()?)) + } + golem_api_grpc::proto::golem::rib::expr::Expr::Not(expr) => { + let expr = expr.expr.ok_or("Missing expr")?; + Expr::Not(Box::new((*expr).try_into()?)) + } + golem_api_grpc::proto::golem::rib::expr::Expr::GreaterThan(expr) => { + let left = expr.left.ok_or("Missing left expr")?; + let right = expr.right.ok_or("Missing right expr")?; + Expr::GreaterThan( + Box::new((*left).try_into()?), + Box::new((*right).try_into()?), + ) + } + golem_api_grpc::proto::golem::rib::expr::Expr::GreaterThanOrEqual(expr) => { + let left = expr.left.ok_or("Missing left expr")?; + let right = expr.right.ok_or("Missing right expr")?; + Expr::GreaterThanOrEqualTo( + Box::new((*left).try_into()?), + Box::new((*right).try_into()?), + ) + } + golem_api_grpc::proto::golem::rib::expr::Expr::LessThan(expr) => { + let left = expr.left.ok_or("Missing left expr")?; + let right = expr.right.ok_or("Missing right expr")?; + Expr::LessThan( + Box::new((*left).try_into()?), + Box::new((*right).try_into()?), + ) + } + golem_api_grpc::proto::golem::rib::expr::Expr::LessThanOrEqual(expr) => { + let left = expr.left.ok_or("Missing left expr")?; + let right = expr.right.ok_or("Missing right expr")?; + Expr::LessThanOrEqualTo( + Box::new((*left).try_into()?), + Box::new((*right).try_into()?), + ) + } + golem_api_grpc::proto::golem::rib::expr::Expr::EqualTo(expr) => { + let left = expr.left.ok_or("Missing left expr")?; + let right = expr.right.ok_or("Missing right expr")?; + Expr::EqualTo( + Box::new((*left).try_into()?), + Box::new((*right).try_into()?), + ) + } + golem_api_grpc::proto::golem::rib::expr::Expr::Cond(expr) => { + let left = expr.left.ok_or("Missing left expr")?; + let cond = expr.cond.ok_or("Missing cond expr")?; + let right = expr.right.ok_or("Missing right expr")?; + Expr::Cond( + Box::new((*left).try_into()?), + Box::new((*cond).try_into()?), + Box::new((*right).try_into()?), + ) + } + golem_api_grpc::proto::golem::rib::expr::Expr::Concat( + golem_api_grpc::proto::golem::rib::ConcatExpr { exprs }, + ) => { + let exprs: Vec = exprs + .into_iter() + .map(|expr| expr.try_into()) + .collect::, _>>()?; + Expr::Concat(exprs) + } + golem_api_grpc::proto::golem::rib::expr::Expr::Multiple( + golem_api_grpc::proto::golem::rib::MultipleExpr { exprs }, + ) => { + let exprs: Vec = exprs + .into_iter() + .map(|expr| expr.try_into()) + .collect::, _>>()?; + Expr::Multiple(exprs) + } + golem_api_grpc::proto::golem::rib::expr::Expr::Sequence( + golem_api_grpc::proto::golem::rib::SequenceExpr { exprs }, + ) => { + let exprs: Vec = exprs + .into_iter() + .map(|expr| expr.try_into()) + .collect::, _>>()?; + Expr::Sequence(exprs) + } + golem_api_grpc::proto::golem::rib::expr::Expr::Tuple( + golem_api_grpc::proto::golem::rib::TupleExpr { exprs }, + ) => { + let exprs: Vec = exprs + .into_iter() + .map(|expr| expr.try_into()) + .collect::, _>>()?; + Expr::Tuple(exprs) + } + golem_api_grpc::proto::golem::rib::expr::Expr::Record( + golem_api_grpc::proto::golem::rib::RecordExpr { fields }, + ) => { + let mut values: Vec<(String, Box)> = vec![]; + for record in fields.into_iter() { + let name = record.name; + let expr = record.expr.ok_or("Missing expr")?; + values.push((name, Box::new(expr.try_into()?))); + } + Expr::Record(values) + } + golem_api_grpc::proto::golem::rib::expr::Expr::Flags( + golem_api_grpc::proto::golem::rib::FlagsExpr { values }, + ) => Expr::Flags(values), + golem_api_grpc::proto::golem::rib::expr::Expr::Literal( + golem_api_grpc::proto::golem::rib::LiteralExpr { value }, + ) => Expr::Literal(value), + golem_api_grpc::proto::golem::rib::expr::Expr::Identifier( + golem_api_grpc::proto::golem::rib::IdentifierExpr { name }, + ) => Expr::Identifier(name), + golem_api_grpc::proto::golem::rib::expr::Expr::Boolean( + golem_api_grpc::proto::golem::rib::BooleanExpr { value }, + ) => Expr::Boolean(value), + golem_api_grpc::proto::golem::rib::expr::Expr::Number(expr) => { + Expr::Number(expr.try_into()?) + } + golem_api_grpc::proto::golem::rib::expr::Expr::SelectField(expr) => { + let expr = *expr; + let field = expr.field; + let expr = *expr.expr.ok_or( + "Mi\ + ssing expr", + )?; + Expr::SelectField(Box::new(expr.try_into()?), field) + } + golem_api_grpc::proto::golem::rib::expr::Expr::SelectIndex(expr) => { + let expr = *expr; + let index = expr.index as usize; + let expr = *expr.expr.ok_or("Missing expr")?; + Expr::SelectIndex(Box::new(expr.try_into()?), index) + } + golem_api_grpc::proto::golem::rib::expr::Expr::Option(expr) => match expr.expr { + Some(expr) => Expr::Option(Some(Box::new((*expr).try_into()?))), + None => Expr::Option(None), + }, + golem_api_grpc::proto::golem::rib::expr::Expr::Result(expr) => { + let result = expr.result.ok_or("Missing result")?; + match result { + golem_api_grpc::proto::golem::rib::result_expr::Result::Ok(expr) => { + Expr::Result(Ok(Box::new((*expr).try_into()?))) + } + golem_api_grpc::proto::golem::rib::result_expr::Result::Err(expr) => { + Expr::Result(Err(Box::new((*expr).try_into()?))) + } + } + } + golem_api_grpc::proto::golem::rib::expr::Expr::PatternMatch(expr) => { + let patterns: Vec = expr + .patterns + .into_iter() + .map(|expr| expr.try_into()) + .collect::, _>>()?; + let expr = expr.expr.ok_or("Missing expr")?; + Expr::PatternMatch(Box::new((*expr).try_into()?), patterns) + } + golem_api_grpc::proto::golem::rib::expr::Expr::Call(expr) => { + let params: Vec = expr + .params + .into_iter() + .map(|expr| expr.try_into()) + .collect::, _>>()?; + let name = expr.name.ok_or("Missing name")?; + Expr::Call(name.try_into()?, params) + } + }; + Ok(expr) + } +} + +impl From for golem_api_grpc::proto::golem::rib::Expr { + fn from(value: Expr) -> Self { + let expr = match value { + Expr::Let(name, expr) => golem_api_grpc::proto::golem::rib::expr::Expr::Let(Box::new( + golem_api_grpc::proto::golem::rib::LetExpr { + name, + expr: Some(Box::new((*expr).into())), + }, + )), + Expr::SelectField(expr, field) => { + golem_api_grpc::proto::golem::rib::expr::Expr::SelectField(Box::new( + golem_api_grpc::proto::golem::rib::SelectFieldExpr { + expr: Some(Box::new((*expr).into())), + field, + }, + )) + } + Expr::SelectIndex(expr, index) => { + golem_api_grpc::proto::golem::rib::expr::Expr::SelectIndex(Box::new( + golem_api_grpc::proto::golem::rib::SelectIndexExpr { + expr: Some(Box::new((*expr).into())), + index: index as u64, + }, + )) + } + Expr::Sequence(exprs) => golem_api_grpc::proto::golem::rib::expr::Expr::Sequence( + golem_api_grpc::proto::golem::rib::SequenceExpr { + exprs: exprs.into_iter().map(|expr| expr.into()).collect(), + }, + ), + Expr::Record(fields) => golem_api_grpc::proto::golem::rib::expr::Expr::Record( + golem_api_grpc::proto::golem::rib::RecordExpr { + fields: fields + .into_iter() + .map( + |(name, expr)| golem_api_grpc::proto::golem::rib::RecordFieldExpr { + name, + expr: Some((*expr).into()), + }, + ) + .collect(), + }, + ), + Expr::Tuple(exprs) => golem_api_grpc::proto::golem::rib::expr::Expr::Tuple( + golem_api_grpc::proto::golem::rib::TupleExpr { + exprs: exprs.into_iter().map(|expr| expr.into()).collect(), + }, + ), + Expr::Literal(value) => golem_api_grpc::proto::golem::rib::expr::Expr::Literal( + golem_api_grpc::proto::golem::rib::LiteralExpr { value }, + ), + Expr::Number(number) => { + golem_api_grpc::proto::golem::rib::expr::Expr::Number(number.into()) + } + Expr::Flags(values) => golem_api_grpc::proto::golem::rib::expr::Expr::Flags( + golem_api_grpc::proto::golem::rib::FlagsExpr { values }, + ), + Expr::Identifier(name) => golem_api_grpc::proto::golem::rib::expr::Expr::Identifier( + golem_api_grpc::proto::golem::rib::IdentifierExpr { name }, + ), + Expr::Boolean(value) => golem_api_grpc::proto::golem::rib::expr::Expr::Boolean( + golem_api_grpc::proto::golem::rib::BooleanExpr { value }, + ), + Expr::Concat(exprs) => golem_api_grpc::proto::golem::rib::expr::Expr::Concat( + golem_api_grpc::proto::golem::rib::ConcatExpr { + exprs: exprs.into_iter().map(|expr| expr.into()).collect(), + }, + ), + Expr::Multiple(exprs) => golem_api_grpc::proto::golem::rib::expr::Expr::Multiple( + golem_api_grpc::proto::golem::rib::MultipleExpr { + exprs: exprs.into_iter().map(|expr| expr.into()).collect(), + }, + ), + Expr::Not(expr) => golem_api_grpc::proto::golem::rib::expr::Expr::Not(Box::new( + golem_api_grpc::proto::golem::rib::NotExpr { + expr: Some(Box::new((*expr).into())), + }, + )), + Expr::GreaterThan(left, right) => { + golem_api_grpc::proto::golem::rib::expr::Expr::GreaterThan(Box::new( + golem_api_grpc::proto::golem::rib::GreaterThanExpr { + left: Some(Box::new((*left).into())), + right: Some(Box::new((*right).into())), + }, + )) + } + Expr::GreaterThanOrEqualTo(left, right) => { + golem_api_grpc::proto::golem::rib::expr::Expr::GreaterThanOrEqual(Box::new( + golem_api_grpc::proto::golem::rib::GreaterThanOrEqualToExpr { + left: Some(Box::new((*left).into())), + right: Some(Box::new((*right).into())), + }, + )) + } + Expr::LessThan(left, right) => golem_api_grpc::proto::golem::rib::expr::Expr::LessThan( + Box::new(golem_api_grpc::proto::golem::rib::LessThanExpr { + left: Some(Box::new((*left).into())), + right: Some(Box::new((*right).into())), + }), + ), + Expr::LessThanOrEqualTo(left, right) => { + golem_api_grpc::proto::golem::rib::expr::Expr::LessThanOrEqual(Box::new( + golem_api_grpc::proto::golem::rib::LessThanOrEqualToExpr { + left: Some(Box::new((*left).into())), + right: Some(Box::new((*right).into())), + }, + )) + } + Expr::EqualTo(left, right) => golem_api_grpc::proto::golem::rib::expr::Expr::EqualTo( + Box::new(golem_api_grpc::proto::golem::rib::EqualToExpr { + left: Some(Box::new((*left).into())), + right: Some(Box::new((*right).into())), + }), + ), + Expr::Cond(left, cond, right) => golem_api_grpc::proto::golem::rib::expr::Expr::Cond( + Box::new(golem_api_grpc::proto::golem::rib::CondExpr { + left: Some(Box::new((*left).into())), + cond: Some(Box::new((*cond).into())), + right: Some(Box::new((*right).into())), + }), + ), + Expr::PatternMatch(expr, arms) => { + golem_api_grpc::proto::golem::rib::expr::Expr::PatternMatch(Box::new( + golem_api_grpc::proto::golem::rib::PatternMatchExpr { + expr: Some(Box::new((*expr).into())), + patterns: arms.into_iter().map(|a| a.into()).collect(), + }, + )) + } + Expr::Option(expr) => golem_api_grpc::proto::golem::rib::expr::Expr::Option(Box::new( + golem_api_grpc::proto::golem::rib::OptionExpr { + expr: expr.map(|expr| Box::new((*expr).into())), + }, + )), + Expr::Result(expr) => { + let result = match expr { + Ok(expr) => golem_api_grpc::proto::golem::rib::result_expr::Result::Ok( + Box::new((*expr).into()), + ), + Err(expr) => golem_api_grpc::proto::golem::rib::result_expr::Result::Err( + Box::new((*expr).into()), + ), + }; + + golem_api_grpc::proto::golem::rib::expr::Expr::Result(Box::new( + golem_api_grpc::proto::golem::rib::ResultExpr { + result: Some(result), + }, + )) + } + Expr::Call(function_name, args) => golem_api_grpc::proto::golem::rib::expr::Expr::Call( + golem_api_grpc::proto::golem::rib::CallExpr { + name: Some(function_name.into()), + params: args.into_iter().map(|expr| expr.into()).collect(), + }, + ), + }; + + golem_api_grpc::proto::golem::rib::Expr { expr: Some(expr) } + } +} + #[derive(Debug, Clone, PartialEq, Encode, Decode)] pub enum Number { Unsigned(u64), @@ -166,9 +509,66 @@ impl Display for Number { } } +impl TryFrom for Number { + type Error = String; + + fn try_from(value: golem_api_grpc::proto::golem::rib::NumberExpr) -> Result { + let number = value.number.ok_or("Missing number")?; + match number { + golem_api_grpc::proto::golem::rib::number_expr::Number::Unsigned(value) => { + Ok(Number::Unsigned(value)) + } + golem_api_grpc::proto::golem::rib::number_expr::Number::Signed(value) => { + Ok(Number::Signed(value)) + } + golem_api_grpc::proto::golem::rib::number_expr::Number::Float(value) => { + Ok(Number::Float(value)) + } + } + } +} + +impl From for golem_api_grpc::proto::golem::rib::NumberExpr { + fn from(value: Number) -> Self { + golem_api_grpc::proto::golem::rib::NumberExpr { + number: Some(match value { + Number::Unsigned(value) => { + golem_api_grpc::proto::golem::rib::number_expr::Number::Unsigned(value) + } + Number::Signed(value) => { + golem_api_grpc::proto::golem::rib::number_expr::Number::Signed(value) + } + Number::Float(value) => { + golem_api_grpc::proto::golem::rib::number_expr::Number::Float(value) + } + }), + } + } +} + #[derive(Debug, Clone, PartialEq, Encode, Decode)] pub struct MatchArm(pub (ArmPattern, Box)); +impl TryFrom for MatchArm { + type Error = String; + + fn try_from(value: golem_api_grpc::proto::golem::rib::MatchArm) -> Result { + let pattern = value.pattern.ok_or("Missing pattern")?; + let expr = value.expr.ok_or("Missing expr")?; + Ok(MatchArm((pattern.try_into()?, Box::new(expr.try_into()?)))) + } +} + +impl From for golem_api_grpc::proto::golem::rib::MatchArm { + fn from(value: MatchArm) -> Self { + let (pattern, expr) = value.0; + golem_api_grpc::proto::golem::rib::MatchArm { + pattern: Some(pattern.into()), + expr: Some((*expr).into()), + } + } +} + // Ex: Some(x) #[derive(Debug, Clone, PartialEq, Encode, Decode)] pub enum ArmPattern { @@ -212,6 +612,85 @@ impl ArmPattern { } } +impl TryFrom for ArmPattern { + type Error = String; + + fn try_from(value: golem_api_grpc::proto::golem::rib::ArmPattern) -> Result { + let pattern = value.pattern.ok_or("Missing pattern")?; + match pattern { + golem_api_grpc::proto::golem::rib::arm_pattern::Pattern::WildCard(_) => { + Ok(ArmPattern::WildCard) + } + golem_api_grpc::proto::golem::rib::arm_pattern::Pattern::As(asp) => { + let name = asp.name; + let pattern = asp.pattern.ok_or("Missing pattern")?; + Ok(ArmPattern::As(name, Box::new((*pattern).try_into()?))) + } + golem_api_grpc::proto::golem::rib::arm_pattern::Pattern::Constructor( + golem_api_grpc::proto::golem::rib::ConstructorArmPattern { name, patterns }, + ) => { + let patterns = patterns + .into_iter() + .map(ArmPattern::try_from) + .collect::, _>>()?; + Ok(ArmPattern::Constructor(name, patterns)) + } + golem_api_grpc::proto::golem::rib::arm_pattern::Pattern::Literal( + golem_api_grpc::proto::golem::rib::LiteralArmPattern { expr }, + ) => { + let inner = expr.ok_or("Missing expr")?; + Ok(ArmPattern::Literal(Box::new(inner.try_into()?))) + } + } + } +} + +impl From for golem_api_grpc::proto::golem::rib::ArmPattern { + fn from(value: ArmPattern) -> Self { + match value { + ArmPattern::WildCard => golem_api_grpc::proto::golem::rib::ArmPattern { + pattern: Some( + golem_api_grpc::proto::golem::rib::arm_pattern::Pattern::WildCard( + golem_api_grpc::proto::golem::rib::WildCardArmPattern {}, + ), + ), + }, + ArmPattern::As(name, pattern) => golem_api_grpc::proto::golem::rib::ArmPattern { + pattern: Some(golem_api_grpc::proto::golem::rib::arm_pattern::Pattern::As( + Box::new(golem_api_grpc::proto::golem::rib::AsArmPattern { + name, + pattern: Some(Box::new((*pattern).into())), + }), + )), + }, + ArmPattern::Constructor(name, patterns) => { + golem_api_grpc::proto::golem::rib::ArmPattern { + pattern: Some( + golem_api_grpc::proto::golem::rib::arm_pattern::Pattern::Constructor( + golem_api_grpc::proto::golem::rib::ConstructorArmPattern { + name, + patterns: patterns + .into_iter() + .map(golem_api_grpc::proto::golem::rib::ArmPattern::from) + .collect(), + }, + ), + ), + } + } + ArmPattern::Literal(expr) => golem_api_grpc::proto::golem::rib::ArmPattern { + pattern: Some( + golem_api_grpc::proto::golem::rib::arm_pattern::Pattern::Literal( + golem_api_grpc::proto::golem::rib::LiteralArmPattern { + expr: Some((*expr).into()), + }, + ), + ), + }, + } + } +} + impl Display for Expr { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", text::to_string(self).unwrap()) diff --git a/golem-rib/src/function_name.rs b/golem-rib/src/function_name.rs index 6eae7dc201..da7f315774 100644 --- a/golem-rib/src/function_name.rs +++ b/golem-rib/src/function_name.rs @@ -106,6 +106,33 @@ impl<'de> BorrowDecode<'de> for SemVer { } } +impl TryFrom for SemVer { + type Error = String; + + fn try_from(value: golem_api_grpc::proto::golem::rib::SemVersion) -> Result { + Ok(SemVer(semver::Version { + major: value.major, + minor: value.minor, + patch: value.patch, + pre: Prerelease::new(&value.pre).map_err(|_| "Invalid prerelease".to_string())?, + build: BuildMetadata::new(&value.build) + .map_err(|_| "Invalid build metadata".to_string())?, + })) + } +} + +impl From for golem_api_grpc::proto::golem::rib::SemVersion { + fn from(value: SemVer) -> Self { + golem_api_grpc::proto::golem::rib::SemVersion { + major: value.0.major, + minor: value.0.minor, + patch: value.0.patch, + pre: value.0.pre.to_string(), + build: value.0.build.to_string(), + } + } +} + impl ParsedFunctionSite { pub fn interface_name(&self) -> Option { match self { @@ -127,6 +154,75 @@ impl ParsedFunctionSite { } } +impl TryFrom for ParsedFunctionSite { + type Error = String; + + fn try_from( + value: golem_api_grpc::proto::golem::rib::ParsedFunctionSite, + ) -> Result { + let site = value.site.ok_or("Missing site".to_string())?; + match site { + golem_api_grpc::proto::golem::rib::parsed_function_site::Site::Global(_) => { + Ok(Self::Global) + } + golem_api_grpc::proto::golem::rib::parsed_function_site::Site::Interface( + golem_api_grpc::proto::golem::rib::InterfaceFunctionSite { name }, + ) => Ok(Self::Interface { name }), + golem_api_grpc::proto::golem::rib::parsed_function_site::Site::PackageInterface( + golem_api_grpc::proto::golem::rib::PackageInterfaceFunctionSite { + namespace, + package, + interface, + version, + }, + ) => { + let version = match version { + Some(version) => Some(version.try_into()?), + None => None, + }; + + Ok(Self::PackagedInterface { + namespace, + package, + interface, + version, + }) + } + } + } +} + +impl From for golem_api_grpc::proto::golem::rib::ParsedFunctionSite { + fn from(value: ParsedFunctionSite) -> Self { + let site = match value { + ParsedFunctionSite::Global => { + golem_api_grpc::proto::golem::rib::parsed_function_site::Site::Global( + golem_api_grpc::proto::golem::rib::GlobalFunctionSite {}, + ) + } + ParsedFunctionSite::Interface { name } => { + golem_api_grpc::proto::golem::rib::parsed_function_site::Site::Interface( + golem_api_grpc::proto::golem::rib::InterfaceFunctionSite { name }, + ) + } + ParsedFunctionSite::PackagedInterface { + namespace, + package, + interface, + version, + } => golem_api_grpc::proto::golem::rib::parsed_function_site::Site::PackageInterface( + golem_api_grpc::proto::golem::rib::PackageInterfaceFunctionSite { + namespace, + package, + interface, + version: version.map(|v| v.into()), + }, + ), + }; + golem_api_grpc::proto::golem::rib::ParsedFunctionSite { site: Some(site) } + } +} + #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] pub enum ParsedFunctionReference { Function { @@ -283,6 +379,168 @@ impl ParsedFunctionReference { } } +impl TryFrom + for ParsedFunctionReference +{ + type Error = String; + + fn try_from( + value: golem_api_grpc::proto::golem::rib::ParsedFunctionReference, + ) -> Result { + let function = value + .function_reference + .ok_or("Missing function".to_string())?; + match function { + golem_api_grpc::proto::golem::rib::parsed_function_reference::FunctionReference::Function(golem_api_grpc::proto::golem::rib::FunctionFunctionReference { + function + }) => { + Ok(Self::Function { function }) + } + golem_api_grpc::proto::golem::rib::parsed_function_reference::FunctionReference::RawResourceConstructor(golem_api_grpc::proto::golem::rib::RawResourceConstructorFunctionReference { + resource + }) => { + Ok(Self::RawResourceConstructor { resource }) + } + golem_api_grpc::proto::golem::rib::parsed_function_reference::FunctionReference::RawResourceMethod(golem_api_grpc::proto::golem::rib::RawResourceMethodFunctionReference { + resource, + method + }) => { + Ok(Self::RawResourceMethod { resource, method }) + } + golem_api_grpc::proto::golem::rib::parsed_function_reference::FunctionReference::RawResourceStaticMethod(golem_api_grpc::proto::golem::rib::RawResourceStaticMethodFunctionReference { + resource, + method + }) => { + Ok(Self::RawResourceStaticMethod { resource, method }) + } + golem_api_grpc::proto::golem::rib::parsed_function_reference::FunctionReference::RawResourceDrop(golem_api_grpc::proto::golem::rib::RawResourceDropFunctionReference { + resource + }) => { + Ok(Self::RawResourceDrop { resource }) + } + golem_api_grpc::proto::golem::rib::parsed_function_reference::FunctionReference::IndexedResourceConstructor(golem_api_grpc::proto::golem::rib::IndexedResourceConstructorFunctionReference { + resource, + resource_params + }) => { + Ok(Self::IndexedResourceConstructor { + resource, + resource_params, + }) + } + golem_api_grpc::proto::golem::rib::parsed_function_reference::FunctionReference::IndexedResourceMethod(golem_api_grpc::proto::golem::rib::IndexedResourceMethodFunctionReference { + resource, + resource_params, + method + }) => { + Ok(Self::IndexedResourceMethod { + resource, + resource_params, + method, + }) + } + golem_api_grpc::proto::golem::rib::parsed_function_reference::FunctionReference::IndexedResourceStaticMethod(golem_api_grpc::proto::golem::rib::IndexedResourceStaticMethodFunctionReference { + resource, + resource_params, + method + }) => { + Ok(Self::IndexedResourceStaticMethod { + resource, + resource_params, + method, + }) + } + golem_api_grpc::proto::golem::rib::parsed_function_reference::FunctionReference::IndexedResourceDrop(golem_api_grpc::proto::golem::rib::IndexedResourceDropFunctionReference { + resource, + resource_params + }) => { + Ok(Self::IndexedResourceDrop { + resource, + resource_params, + }) + } + } + } +} + +impl From for golem_api_grpc::proto::golem::rib::ParsedFunctionReference { + fn from(value: ParsedFunctionReference) -> Self { + let function = match value { + ParsedFunctionReference::Function { function } => golem_api_grpc::proto::golem::rib::parsed_function_reference::FunctionReference::Function( + golem_api_grpc::proto::golem::rib::FunctionFunctionReference { function }, + ), + ParsedFunctionReference::RawResourceConstructor { resource } => { + golem_api_grpc::proto::golem::rib::parsed_function_reference::FunctionReference::RawResourceConstructor( + golem_api_grpc::proto::golem::rib::RawResourceConstructorFunctionReference { + resource, + }, + ) + } + ParsedFunctionReference::RawResourceMethod { resource, method } => { + golem_api_grpc::proto::golem::rib::parsed_function_reference::FunctionReference::RawResourceMethod( + golem_api_grpc::proto::golem::rib::RawResourceMethodFunctionReference { + resource, + method, + }, + ) + } + ParsedFunctionReference::RawResourceStaticMethod { resource, method } => { + golem_api_grpc::proto::golem::rib::parsed_function_reference::FunctionReference::RawResourceStaticMethod( + golem_api_grpc::proto::golem::rib::RawResourceStaticMethodFunctionReference { + resource, + method, + }, + ) + } + ParsedFunctionReference::RawResourceDrop { resource } => golem_api_grpc::proto::golem::rib::parsed_function_reference::FunctionReference::RawResourceDrop( + golem_api_grpc::proto::golem::rib::RawResourceDropFunctionReference { resource }, + ), + ParsedFunctionReference::IndexedResourceConstructor { + resource, + resource_params, + } => golem_api_grpc::proto::golem::rib::parsed_function_reference::FunctionReference::IndexedResourceConstructor( + golem_api_grpc::proto::golem::rib::IndexedResourceConstructorFunctionReference { + resource, + resource_params, + }, + ), + ParsedFunctionReference::IndexedResourceMethod { + resource, + resource_params, + method, + } => golem_api_grpc::proto::golem::rib::parsed_function_reference::FunctionReference::IndexedResourceMethod( + golem_api_grpc::proto::golem::rib::IndexedResourceMethodFunctionReference { + resource, + resource_params, + method, + }, + ), + ParsedFunctionReference::IndexedResourceStaticMethod { + resource, + resource_params, + method, + } => golem_api_grpc::proto::golem::rib::parsed_function_reference::FunctionReference::IndexedResourceStaticMethod( + golem_api_grpc::proto::golem::rib::IndexedResourceStaticMethodFunctionReference { + resource, + resource_params, + method, + }, + ), + ParsedFunctionReference::IndexedResourceDrop { + resource, + resource_params, + } => golem_api_grpc::proto::golem::rib::parsed_function_reference::FunctionReference::IndexedResourceDrop( + golem_api_grpc::proto::golem::rib::IndexedResourceDropFunctionReference { + resource, + resource_params, + }, + ), + }; + golem_api_grpc::proto::golem::rib::ParsedFunctionReference { + function_reference: Some(function), + } + } +} + #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] pub struct ParsedFunctionName { pub site: ParsedFunctionSite, @@ -343,6 +601,29 @@ impl ParsedFunctionName { } } +impl TryFrom for ParsedFunctionName { + type Error = String; + + fn try_from( + value: golem_api_grpc::proto::golem::rib::ParsedFunctionName, + ) -> Result { + let site = ParsedFunctionSite::try_from(value.site.ok_or("Missing site".to_string())?)?; + let function = ParsedFunctionReference::try_from( + value.function.ok_or("Missing function".to_string())?, + )?; + Ok(Self { site, function }) + } +} + +impl From for golem_api_grpc::proto::golem::rib::ParsedFunctionName { + fn from(value: ParsedFunctionName) -> Self { + golem_api_grpc::proto::golem::rib::ParsedFunctionName { + site: Some(value.site.into()), + function: Some(value.function.into()), + } + } +} + #[cfg(test)] mod function_name_tests { use super::{ParsedFunctionName, ParsedFunctionReference, ParsedFunctionSite, SemVer}; @@ -358,7 +639,7 @@ mod function_name_tests { site: ParsedFunctionSite::Global, function: ParsedFunctionReference::Function { function: "run-example".to_string() - } + }, } ); } @@ -380,7 +661,7 @@ mod function_name_tests { }, function: ParsedFunctionReference::Function { function: "fn1".to_string() - } + }, } ); } @@ -400,11 +681,11 @@ mod function_name_tests { namespace: "ns".to_string(), package: "name".to_string(), interface: "interface".to_string(), - version: None + version: None, }, function: ParsedFunctionReference::Function { function: "fn1".to_string() - } + }, } ); } @@ -424,11 +705,11 @@ mod function_name_tests { namespace: "wasi".to_string(), package: "cli".to_string(), interface: "run".to_string(), - version: Some(SemVer(semver::Version::new(0, 2, 0))) + version: Some(SemVer(semver::Version::new(0, 2, 0))), }, function: ParsedFunctionReference::Function { function: "run".to_string() - } + }, } ); } @@ -452,11 +733,11 @@ mod function_name_tests { namespace: "ns".to_string(), package: "name".to_string(), interface: "interface".to_string(), - version: None + version: None, }, function: ParsedFunctionReference::RawResourceConstructor { resource: "resource1".to_string() - } + }, } ); } @@ -480,11 +761,11 @@ mod function_name_tests { namespace: "ns".to_string(), package: "name".to_string(), interface: "interface".to_string(), - version: None + version: None, }, function: ParsedFunctionReference::RawResourceConstructor { resource: "resource1".to_string() - } + }, } ); } @@ -510,12 +791,12 @@ mod function_name_tests { namespace: "ns".to_string(), package: "name".to_string(), interface: "interface".to_string(), - version: None + version: None, }, function: ParsedFunctionReference::IndexedResourceConstructor { resource: "resource1".to_string(), - resource_params: vec![] - } + resource_params: vec![], + }, } ); } @@ -624,12 +905,12 @@ mod function_name_tests { namespace: "ns".to_string(), package: "name".to_string(), interface: "interface".to_string(), - version: None + version: None, }, function: ParsedFunctionReference::RawResourceMethod { resource: "resource1".to_string(), - method: "do-something".to_string() - } + method: "do-something".to_string(), + }, } ); } @@ -654,12 +935,12 @@ mod function_name_tests { namespace: "ns".to_string(), package: "name".to_string(), interface: "interface".to_string(), - version: None + version: None, }, function: ParsedFunctionReference::RawResourceMethod { resource: "resource1".to_string(), - method: "do-something".to_string() - } + method: "do-something".to_string(), + }, } ); } @@ -687,12 +968,12 @@ mod function_name_tests { namespace: "ns".to_string(), package: "name".to_string(), interface: "interface".to_string(), - version: None + version: None, }, function: ParsedFunctionReference::RawResourceStaticMethod { resource: "resource1".to_string(), - method: "do-something-static".to_string() - } + method: "do-something-static".to_string(), + }, } ); } @@ -717,12 +998,12 @@ mod function_name_tests { namespace: "ns".to_string(), package: "name".to_string(), interface: "interface".to_string(), - version: None + version: None, }, function: ParsedFunctionReference::RawResourceStaticMethod { resource: "resource1".to_string(), - method: "do-something-static".to_string() - } + method: "do-something-static".to_string(), + }, } ); } @@ -746,11 +1027,11 @@ mod function_name_tests { namespace: "ns".to_string(), package: "name".to_string(), interface: "interface".to_string(), - version: None + version: None, }, function: ParsedFunctionReference::RawResourceDrop { resource: "resource1".to_string() - } + }, } ); } @@ -776,12 +1057,12 @@ mod function_name_tests { namespace: "ns".to_string(), package: "name".to_string(), interface: "interface".to_string(), - version: None + version: None, }, function: ParsedFunctionReference::IndexedResourceDrop { resource: "resource1".to_string(), - resource_params: vec![] - } + resource_params: vec![], + }, } ) } @@ -815,7 +1096,7 @@ mod function_name_tests { namespace: "ns".to_string(), package: "name".to_string(), interface: "interface".to_string(), - version: None + version: None, }, function: ParsedFunctionReference::IndexedResourceDrop { resource: "resource1".to_string(), @@ -823,8 +1104,8 @@ mod function_name_tests { "\"hello\"".to_string(), "1".to_string(), "true".to_string(), - ] - } + ], + }, } ); } @@ -890,11 +1171,11 @@ mod function_name_tests { namespace: "ns".to_string(), package: "name".to_string(), interface: "interface".to_string(), - version: None + version: None, }, function: ParsedFunctionReference::RawResourceDrop { resource: "resource1".to_string() - } + }, } ); } From b1c5599d2d777de1c5268931ef379afae2c04b6b Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Fri, 5 Jul 2024 22:17:35 +0200 Subject: [PATCH 23/30] WorkerBinding proto - golem.rib.Expr --- .../golem/apidefinition/api_definition.proto | 7 ++++--- .../src/api/register_api_definition_api.rs | 21 +++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/golem-api-grpc/proto/golem/apidefinition/api_definition.proto b/golem-api-grpc/proto/golem/apidefinition/api_definition.proto index f0f99166df..9faf4d1852 100644 --- a/golem-api-grpc/proto/golem/apidefinition/api_definition.proto +++ b/golem-api-grpc/proto/golem/apidefinition/api_definition.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package golem.apidefinition; import "golem/component/component_id.proto"; +import "golem/rib/expr.proto"; message ApiDefinition { ApiDefinitionId id = 1; @@ -45,7 +46,7 @@ enum HttpMethod { message WorkerBinding { golem.component.ComponentId component = 1; - string worker_name = 2; - string response = 3; - optional string idempotency_key = 4; + golem.rib.Expr worker_name = 2; + golem.rib.Expr response = 3; + optional golem.rib.Expr idempotency_key = 4; } \ No newline at end of file diff --git a/golem-worker-service-base/src/api/register_api_definition_api.rs b/golem-worker-service-base/src/api/register_api_definition_api.rs index 09362b5454..135dbc51af 100644 --- a/golem-worker-service-base/src/api/register_api_definition_api.rs +++ b/golem-worker-service-base/src/api/register_api_definition_api.rs @@ -309,15 +309,11 @@ impl TryFrom for grpc_apidefinition:: type Error = String; fn try_from(value: crate::worker_binding::GolemWorkerBinding) -> Result { - let response: String = value.response.0.to_string(); + let response = Some(value.response.0.into()); - let worker_name = rib::to_string(&value.worker_name).map_err(|e| e.to_string())?; + let worker_name = Some(value.worker_name.into()); - let idempotency_key = if let Some(key) = &value.idempotency_key { - Some(rib::to_string(key).map_err(|e| e.to_string())?) - } else { - None - }; + let idempotency_key = value.idempotency_key.map(|key| key.into()); let result = grpc_apidefinition::WorkerBinding { component: Some(value.component_id.into()), @@ -335,16 +331,19 @@ impl TryFrom for crate::worker_binding::Golem fn try_from(value: grpc_apidefinition::WorkerBinding) -> Result { let response: crate::worker_binding::ResponseMapping = { - let r: Expr = value.response.parse()?; + let r: Expr = value.response.ok_or("response is missing")?.try_into()?; crate::worker_binding::ResponseMapping(r) }; - let worker_name = value.worker_name.parse()?; + let worker_name = value + .worker_name + .ok_or("worker name is missing")? + .try_into()?; let component_id = value.component.ok_or("component is missing")?.try_into()?; - let idempotency_key = if let Some(key) = &value.idempotency_key { - Some(key.parse()?) + let idempotency_key = if let Some(key) = value.idempotency_key { + Some(key.try_into()?) } else { None }; From 37dd564885af1147c2c01149bd259999009bc06f Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Sat, 6 Jul 2024 21:38:51 +0200 Subject: [PATCH 24/30] tests --- .../http/http_api_definition.rs | 59 ++++++++- .../tests/services_tests.rs | 120 +++++++++++++----- 2 files changed, 142 insertions(+), 37 deletions(-) diff --git a/golem-worker-service-base/src/api_definition/http/http_api_definition.rs b/golem-worker-service-base/src/api_definition/http/http_api_definition.rs index b088a14ab1..33c8562f93 100644 --- a/golem-worker-service-base/src/api_definition/http/http_api_definition.rs +++ b/golem-worker-service-base/src/api_definition/http/http_api_definition.rs @@ -262,9 +262,9 @@ pub struct Route { #[cfg(test)] mod tests { - use golem_common::serialization; - use super::*; + use golem_api_grpc::proto::golem::apidefinition as grpc_apidefinition; + use golem_common::serialization; #[test] fn split_path_works_with_single_value() { @@ -463,11 +463,10 @@ mod tests { binding: componentId: 0b6d9cd8-f373-4e29-8a5a-548e61b868a5 workerName: '{}' - functionName: golem:it/api/get-cart-contents + functionName: golem:it/api.{{get-cart-contents}} functionParams: {} response: '{}' - "#, path_pattern, worker_id, function_params, response_mapping ); @@ -581,4 +580,56 @@ mod tests { "{status : 200}", ); } + + #[test] + fn test_api_spec_proto_conversion() { + fn test_encode_decode( + path_pattern: &str, + worker_id: &str, + function_params: &str, + response_mapping: &str, + ) { + let yaml = get_api_spec(path_pattern, worker_id, function_params, response_mapping); + let original: HttpApiDefinition = serde_yaml::from_value(yaml.clone()).unwrap(); + + let proto: grpc_apidefinition::ApiDefinition = original.clone().try_into().unwrap(); + let decoded: HttpApiDefinition = proto.try_into().unwrap(); + assert_eq!(original, decoded); + } + + test_encode_decode( + "foo/{user-id}", + "shopping-cart-${if (${request.path.user-id}>100) then 0 else 1}", + "[\"${request.body}\"]", + "{status : 200}", + ); + + test_encode_decode( + "foo/{user-id}", + "shopping-cart-${if (${request.path.user-id}>100) then 0 else 1}", + "[\"${request.body.foo}\"]", + "{status : 200}", + ); + + test_encode_decode( + "foo/{user-id}", + "shopping-cart-${if (${request.path.user-id}>100) then 0 else 1}", + "[\"${request.path.user-id}\"]", + "{status : 200}", + ); + + test_encode_decode( + "foo", + "shopping-cart-${if (${request.body.user-id}>100) then 0 else 1}", + "[ \"data\"]", + "{status : 200}", + ); + + test_encode_decode( + "foo", + "match worker.response { ok(value) => 1, error => 0 }", + "[ \"data\"]", + "{status : 200}", + ); + } } diff --git a/golem-worker-service-base/tests/services_tests.rs b/golem-worker-service-base/tests/services_tests.rs index 20f1001ce9..fc6a023710 100644 --- a/golem-worker-service-base/tests/services_tests.rs +++ b/golem-worker-service-base/tests/services_tests.rs @@ -135,7 +135,7 @@ mod tests { api_definition_repo.clone(), )); - test_definition_create_and_delete(definition_service.clone()).await; + test_definition_crud(definition_service.clone()).await; test_deployment(definition_service.clone(), deployment_service.clone()).await; test_deployment_conflict(definition_service.clone(), deployment_service.clone()).await; } @@ -151,19 +151,19 @@ mod tests { let def1 = get_api_definition( &Uuid::new_v4().to_string(), "0.0.1", - "/api/get1", - "worker1", - "[]", - "[]", + "/api/1/foo/{user-id}", + "shopping-cart-${if (${request.path.user-id}>100) then 0 else 1}", + "[\"${request.path.user-id}\"]", + "{status: if (worker.response.user == admin) then 401 else 200}", false, ); let def2draft = get_api_definition( &Uuid::new_v4().to_string(), "0.0.1", - "/api/get2", - "worker2", - "[]", - "[]", + "/api/2/foo/{user-id}", + "shopping-cart-${if (${request.body.user-id}>100) then 0 else 1}", + "[ \"data\"]", + "{status: if (worker.response.user == admin) then 401 else 200}", true, ); let def2 = HttpApiDefinition { @@ -173,19 +173,19 @@ mod tests { let def3 = get_api_definition( &Uuid::new_v4().to_string(), "0.0.1", - "/api/get3", - "worker3", - "[]", - "[]", + "/api/3/foo/{user-id}?{id}", + "shopping-cart-${if (${request.path.user-id}>100) then 0 else 1}", + "[\"${request.body}\"]", + "{status: if (worker.response.user == admin) then 401 else 200}", false, ); let def4 = get_api_definition( &Uuid::new_v4().to_string(), "0.0.1", - "/api/get4", - "worker4", - "[]", - "[]", + "/api/4/foo/{user-id}", + "shopping-cart-${if (${request.path.user-id}>100) then 0 else 1}", + "[\"${request.path.user-id}\"]", + "{status: if (worker.response.user == admin) then 401 else 200}", false, ); @@ -420,35 +420,54 @@ mod tests { ); } - async fn test_definition_create_and_delete( + async fn test_definition_crud( definition_service: Arc< dyn ApiDefinitionService + Sync + Send, >, ) { - let def1 = get_api_definition( + let def1v1 = get_api_definition( &Uuid::new_v4().to_string(), "0.0.1", "/api/get1", - "worker1", - "[]", - "[]", + "shopping-cart-${if (${request.path.user-id}>100) then 0 else 1}", + "[\"${request.body.foo}\"]", + "{status: if (worker.response.user == admin) then 401 else 200}", + false, + ); + let def1v1_upd = get_api_definition( + &def1v1.id.0, + "0.0.1", + "/api/get1/1", + "shopping-cart-${if (${request.path.user-id}>100) then 0 else 1}", + "[\"${request.body.foo}\"]", + "{status: if (worker.response.user == admin) then 401 else 200}", false, ); let def1v2 = get_api_definition( - &def1.id.0, + &def1v1.id.0, "0.0.2", "/api/get1/2", - "worker1", - "[]", - "[]", + "shopping-cart-${if (${request.path.user-id}>100) then 0 else 1}", + "[\"${request.body.foo}\"]", + "{status: if (worker.response.user == admin) then 401 else 200}", + true, + ); + + let def1v2_upd = get_api_definition( + &def1v1.id.0, + "0.0.2", + "/api/get1/22", + "shopping-cart-${if (${request.path.user-id}>100) then 0 else 1}", + "[\"${request.body.foo}\"]", + "{status: if (worker.response.user == admin) then 401 else 200}", true, ); definition_service .create( - &def1, + &def1v1, &DefaultNamespace::default(), &EmptyAuthCtx::default(), ) @@ -465,19 +484,54 @@ mod tests { let definitions = definition_service .get_all_versions( - &def1.id, + &def1v1.id, &DefaultNamespace::default(), &EmptyAuthCtx::default(), ) .await .unwrap(); assert_eq!(definitions.len(), 2); - assert!(definitions.contains(&def1) && definitions.contains(&def1v2)); + assert!(definitions.contains(&def1v1) && definitions.contains(&def1v2)); + + let update_result = definition_service + .update( + &def1v1_upd, + &DefaultNamespace::default(), + &EmptyAuthCtx::default(), + ) + .await; + + assert!(update_result.is_err()); + assert_eq!( + update_result.unwrap_err().to_string(), + ApiDefinitionError::::ApiDefinitionNotDraft(def1v1_upd.id) + .to_string() + ); + + let update_result = definition_service + .update( + &def1v2_upd, + &DefaultNamespace::default(), + &EmptyAuthCtx::default(), + ) + .await; + assert!(update_result.is_ok()); + + let definitions = definition_service + .get_all_versions( + &def1v1.id, + &DefaultNamespace::default(), + &EmptyAuthCtx::default(), + ) + .await + .unwrap(); + assert_eq!(definitions.len(), 2); + assert!(definitions.contains(&def1v1) && definitions.contains(&def1v2_upd)); definition_service .delete( - &def1.id, - &def1.version, + &def1v1.id, + &def1v1.version, &DefaultNamespace::default(), &EmptyAuthCtx::default(), ) @@ -495,7 +549,7 @@ mod tests { let definitions = definition_service .get_all_versions( - &def1.id, + &def1v1.id, &DefaultNamespace::default(), &EmptyAuthCtx::default(), ) @@ -547,7 +601,7 @@ mod tests { binding: componentId: 0b6d9cd8-f373-4e29-8a5a-548e61b868a5 workerName: '{}' - functionName: golem:it/api/get-cart-contents + functionName: golem:it/api.{{get-cart-contents}} functionParams: {} response: '{}' "#, From ac5baec276a0f57d14ca686ae080731efac60184 Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Sun, 7 Jul 2024 21:06:57 +0200 Subject: [PATCH 25/30] component download - namespace check --- .../src/service/component.rs | 37 ++++++++++++++----- .../tests/services_tests.rs | 20 ++++++++++ 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/golem-component-service-base/src/service/component.rs b/golem-component-service-base/src/service/component.rs index 2ed1eb81ca..2a1a9e5e99 100644 --- a/golem-component-service-base/src/service/component.rs +++ b/golem-component-service-base/src/service/component.rs @@ -45,7 +45,7 @@ pub enum ComponentError { impl ComponentError { fn internal(error: E, context: C) -> Self where - E: Display + std::fmt::Debug + Send + Sync + 'static, + E: Display + Debug + Send + Sync + 'static, C: Display + Send + Sync + 'static, { ComponentError::Internal(anyhow::Error::msg(error).context(context)) @@ -295,10 +295,19 @@ where ) -> Result, ComponentError> { let versioned_component_id = { match version { - Some(version) => VersionedComponentId { - component_id: component_id.clone(), - version, - }, + Some(version) => { + let stored_namespace: Option = + self.get_namespace(component_id).await?; + match stored_namespace { + Some(stored_namespace) if stored_namespace == namespace.clone() => { + VersionedComponentId { + component_id: component_id.clone(), + version, + } + } + _ => return Err(ComponentError::UnknownComponentId(component_id.clone())), + } + } None => self .component_repo .get_latest_version(namespace.to_string().as_str(), &component_id.0) @@ -331,10 +340,19 @@ where ) -> Result { let versioned_component_id = { match version { - Some(version) => VersionedComponentId { - component_id: component_id.clone(), - version, - }, + Some(version) => { + let stored_namespace: Option = + self.get_namespace(component_id).await?; + match stored_namespace { + Some(stored_namespace) if stored_namespace == namespace.clone() => { + VersionedComponentId { + component_id: component_id.clone(), + version, + } + } + _ => return Err(ComponentError::UnknownComponentId(component_id.clone())), + } + } None => self .component_repo .get_latest_version(namespace.to_string().as_str(), &component_id.0) @@ -351,7 +369,6 @@ where let id = ProtectedComponentId { versioned_component_id, }; - let stream = self .object_store .get_stream(&self.get_protected_object_store_key(&id)) diff --git a/golem-component-service-base/tests/services_tests.rs b/golem-component-service-base/tests/services_tests.rs index fadb813032..acdb6dff80 100644 --- a/golem-component-service-base/tests/services_tests.rs +++ b/golem-component-service-base/tests/services_tests.rs @@ -224,6 +224,26 @@ mod tests { assert!(component2_result.is_some()); assert_eq!(component2_result.unwrap(), DefaultNamespace::default()); + let component1_result = component_service + .download( + &component1v2.versioned_component_id.component_id, + Some(component1v2.versioned_component_id.version), + &DefaultNamespace::default(), + ) + .await + .unwrap(); + assert!(!component1_result.is_empty()); + + let component2_result = component_service + .download( + &component2.versioned_component_id.component_id, + None, + &DefaultNamespace::default(), + ) + .await + .unwrap(); + assert!(!component2_result.is_empty()); + let component1_result = component_service .find_id_by_name(&component1.component_name, &DefaultNamespace::default()) .await From de451784bf242c0ea152b099c5b3c77167527843 Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Sun, 7 Jul 2024 22:49:33 +0200 Subject: [PATCH 26/30] component with namespace --- golem-component-service-base/src/lib.rs | 1 + golem-component-service-base/src/model.rs | 61 +++++++++++++++ .../src/repo/component.rs | 53 ++++++++----- .../src/service/component.rs | 76 +++++++++++-------- .../tests/services_tests.rs | 48 +++++++----- golem-component-service/src/api/component.rs | 12 +-- 6 files changed, 176 insertions(+), 75 deletions(-) create mode 100644 golem-component-service-base/src/model.rs diff --git a/golem-component-service-base/src/lib.rs b/golem-component-service-base/src/lib.rs index 3586d7f8a0..22a61af734 100644 --- a/golem-component-service-base/src/lib.rs +++ b/golem-component-service-base/src/lib.rs @@ -14,5 +14,6 @@ pub mod api; pub mod config; +pub mod model; pub mod repo; pub mod service; diff --git a/golem-component-service-base/src/model.rs b/golem-component-service-base/src/model.rs new file mode 100644 index 0000000000..6480884ee0 --- /dev/null +++ b/golem-component-service-base/src/model.rs @@ -0,0 +1,61 @@ +use golem_service_base::model::{ + ComponentMetadata, ComponentName, ProtectedComponentId, UserComponentId, VersionedComponentId, +}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Component { + pub namespace: Namespace, + pub versioned_component_id: VersionedComponentId, + pub user_component_id: UserComponentId, + pub protected_component_id: ProtectedComponentId, + pub component_name: ComponentName, + pub component_size: u64, + pub metadata: ComponentMetadata, +} + +impl Component { + pub fn next_version(self) -> Self { + let new_version = VersionedComponentId { + component_id: self.versioned_component_id.component_id, + version: self.versioned_component_id.version + 1, + }; + Self { + versioned_component_id: new_version.clone(), + user_component_id: UserComponentId { + versioned_component_id: new_version.clone(), + }, + protected_component_id: ProtectedComponentId { + versioned_component_id: new_version, + }, + ..self + } + } +} + +impl From> for golem_service_base::model::Component { + fn from(value: Component) -> Self { + Self { + versioned_component_id: value.versioned_component_id, + user_component_id: value.user_component_id, + protected_component_id: value.protected_component_id, + component_name: value.component_name, + component_size: value.component_size, + metadata: value.metadata, + } + } +} + +impl From> for golem_api_grpc::proto::golem::component::Component { + fn from(value: Component) -> Self { + Self { + versioned_component_id: Some(value.versioned_component_id.into()), + user_component_id: Some(value.user_component_id.into()), + protected_component_id: Some(value.protected_component_id.into()), + component_name: value.component_name.0, + component_size: value.component_size, + metadata: Some(value.metadata.into()), + project_id: None, + } + } +} diff --git a/golem-component-service-base/src/repo/component.rs b/golem-component-service-base/src/repo/component.rs index d8f994ebce..dffb7c3d06 100644 --- a/golem-component-service-base/src/repo/component.rs +++ b/golem-component-service-base/src/repo/component.rs @@ -19,11 +19,14 @@ use std::sync::Arc; use async_trait::async_trait; use golem_common::model::ComponentId; +use golem_service_base::model::{ + ComponentMetadata, ComponentName, ProtectedComponentId, UserComponentId, VersionedComponentId, +}; use sqlx::{Database, Pool, Row}; use uuid::Uuid; +use crate::model::Component; use crate::repo::RepoError; -use golem_service_base::model::*; #[derive(sqlx::FromRow, Debug, Clone)] pub struct ComponentRecord { @@ -35,7 +38,11 @@ pub struct ComponentRecord { pub metadata: Vec, } -impl TryFrom for Component { +impl TryFrom for Component +where + Namespace: Display + TryFrom + Eq + Clone + Send + Sync, + >::Error: Display + Send + Sync + 'static, +{ type Error = String; fn try_from(value: ComponentRecord) -> Result { let metadata: ComponentMetadata = record_metadata_serde::deserialize(&value.metadata)?; @@ -49,7 +56,9 @@ impl TryFrom for Component { let user_component_id: UserComponentId = UserComponentId { versioned_component_id: versioned_component_id.clone(), }; + let namespace = Namespace::try_from(value.namespace).map_err(|e| e.to_string())?; Ok(Component { + namespace, component_name: ComponentName(value.name), component_size: value.size as u64, metadata, @@ -69,18 +78,20 @@ impl From for VersionedComponentId { } } -impl ComponentRecord { - pub fn new( - namespace: Namespace, - component: Component, - ) -> Result { - let metadata = record_metadata_serde::serialize(&component.metadata)?; +impl TryFrom> for ComponentRecord +where + Namespace: Display, +{ + type Error = String; + + fn try_from(value: Component) -> Result { + let metadata = record_metadata_serde::serialize(&value.metadata)?; Ok(Self { - namespace: namespace.to_string(), - component_id: component.versioned_component_id.component_id.0, - name: component.component_name.0, - size: component.component_size as i32, - version: component.versioned_component_id.version as i64, + namespace: value.namespace.to_string(), + component_id: value.versioned_component_id.component_id.0, + name: value.component_name.0, + size: value.component_size as i32, + version: value.versioned_component_id.version as i64, metadata: metadata.into(), }) } @@ -343,10 +354,10 @@ impl ComponentRepo for DbComponentRepo { WHERE component_id IN (SELECT component_id FROM components WHERE namespace = $1 AND component_id = $2) "# ) - .bind(namespace) - .bind(component_id) - .execute(&mut *transaction) - .await?; + .bind(namespace) + .bind(component_id) + .execute(&mut *transaction) + .await?; sqlx::query("DELETE FROM components WHERE namespace = $1 AND component_id = $2") .bind(namespace) @@ -568,10 +579,10 @@ impl ComponentRepo for DbComponentRepo { WHERE component_id IN (SELECT component_id FROM components WHERE namespace = $1 AND component_id = $2) "# ) - .bind(namespace) - .bind(component_id) - .execute(&mut *transaction) - .await?; + .bind(namespace) + .bind(component_id) + .execute(&mut *transaction) + .await?; sqlx::query("DELETE FROM components WHERE namespace = $1 AND component_id = $2") .bind(namespace) diff --git a/golem-component-service-base/src/service/component.rs b/golem-component-service-base/src/service/component.rs index 2a1a9e5e99..ea68a422a7 100644 --- a/golem-component-service-base/src/service/component.rs +++ b/golem-component-service-base/src/service/component.rs @@ -22,9 +22,12 @@ use golem_common::model::ComponentId; use tap::TapFallible; use tracing::{error, info}; -use crate::repo::component::{ComponentRecord, ComponentRepo}; +use crate::model::Component; +use crate::repo::component::ComponentRepo; use crate::repo::RepoError; -use golem_service_base::model::*; +use golem_service_base::model::{ + ComponentMetadata, ComponentName, ProtectedComponentId, UserComponentId, VersionedComponentId, +}; use golem_service_base::service::component_object_store::ComponentObjectStore; use golem_service_base::stream::ByteStream; @@ -58,11 +61,15 @@ impl From for ComponentError { } } -pub fn create_new_component( +pub fn create_new_component( component_id: &ComponentId, component_name: &ComponentName, data: &[u8], -) -> Result { + namespace: &Namespace, +) -> Result, ComponentProcessingError> +where + Namespace: Eq + Clone + Send + Sync, +{ let metadata = process_component(data)?; let versioned_component_id = VersionedComponentId { @@ -78,6 +85,7 @@ pub fn create_new_component( }; Ok(Component { + namespace: namespace.clone(), component_name: component_name.clone(), component_size: data.len() as u64, metadata, @@ -95,14 +103,14 @@ pub trait ComponentService { component_name: &ComponentName, data: Vec, namespace: &Namespace, - ) -> Result; + ) -> Result, ComponentError>; async fn update( &self, component_id: &ComponentId, data: Vec, namespace: &Namespace, - ) -> Result; + ) -> Result, ComponentError>; async fn download( &self, @@ -129,7 +137,7 @@ pub trait ComponentService { &self, component_name: Option, namespace: &Namespace, - ) -> Result, ComponentError>; + ) -> Result>, ComponentError>; async fn find_id_by_name( &self, @@ -141,19 +149,19 @@ pub trait ComponentService { &self, component_id: &VersionedComponentId, namespace: &Namespace, - ) -> Result, ComponentError>; + ) -> Result>, ComponentError>; async fn get_latest_version( &self, component_id: &ComponentId, namespace: &Namespace, - ) -> Result, ComponentError>; + ) -> Result>, ComponentError>; async fn get( &self, component_id: &ComponentId, namespace: &Namespace, - ) -> Result, ComponentError>; + ) -> Result>, ComponentError>; async fn get_namespace( &self, @@ -193,7 +201,7 @@ where component_name: &ComponentName, data: Vec, namespace: &Namespace, - ) -> Result { + ) -> Result, ComponentError> { info!( "Creating component - namespace: {}, id: {}, name: {}", namespace, @@ -205,7 +213,7 @@ where .await? .map_or(Ok(()), |id| Err(ComponentError::AlreadyExists(id)))?; - let component = create_new_component(component_id, component_name, &data)?; + let component = create_new_component(component_id, component_name, &data, namespace)?; info!( "Uploaded component - namespace: {}, id: {}, version: 0, exports {:?}", @@ -216,7 +224,9 @@ where self.upload_protected_component(&component.protected_component_id, data) )?; - let record = ComponentRecord::new(namespace, component.clone()) + let record = component + .clone() + .try_into() .map_err(|e| ComponentError::internal(e, "Failed to convert record"))?; self.component_repo.create(&record).await?; @@ -233,7 +243,7 @@ where component_id: &ComponentId, data: Vec, namespace: &Namespace, - ) -> Result { + ) -> Result, ComponentError> { info!( "Updating component - namespace: {}, id: {}", namespace, component_id @@ -275,7 +285,9 @@ where metadata, ..next_component }; - let record = ComponentRecord::new(namespace, component.clone()) + let record = component + .clone() + .try_into() .map_err(|e| ComponentError::internal(e, "Failed to convert record"))?; self.component_repo.create(&record).await?; @@ -445,7 +457,7 @@ where &self, component_name: Option, namespace: &Namespace, - ) -> Result, ComponentError> { + ) -> Result>, ComponentError> { let cn = component_name.clone().map_or("N/A".to_string(), |n| n.0); info!( "Find component by name - namespace: {}, name: {}", @@ -465,10 +477,10 @@ where } }; - let values: Vec = records + let values: Vec> = records .iter() .map(|d| d.clone().try_into()) - .collect::, _>>() + .collect::>, _>>() .map_err(|e| ComponentError::internal(e, "Failed to convert record".to_string()))?; Ok(values) @@ -478,7 +490,7 @@ where &self, component_id: &ComponentId, namespace: &Namespace, - ) -> Result, ComponentError> { + ) -> Result>, ComponentError> { info!( "Getting component - namespace: {}, id: {}", namespace, component_id @@ -488,10 +500,10 @@ where .get(namespace.to_string().as_str(), &component_id.0) .await?; - let values: Vec = records + let values: Vec> = records .iter() .map(|d| d.clone().try_into()) - .collect::, _>>() + .collect::>, _>>() .map_err(|e| ComponentError::internal(e, "Failed to convert record".to_string()))?; Ok(values) @@ -501,7 +513,7 @@ where &self, component_id: &VersionedComponentId, namespace: &Namespace, - ) -> Result, ComponentError> { + ) -> Result>, ComponentError> { info!( "Getting component - namespace: {}, id: {}, version: {}", namespace, component_id.component_id, component_id.version @@ -531,7 +543,7 @@ where &self, component_id: &ComponentId, namespace: &Namespace, - ) -> Result, ComponentError> { + ) -> Result>, ComponentError> { info!( "Getting component - namespace: {}, id: {}, version: latest", namespace, component_id @@ -628,9 +640,10 @@ impl + Eq + Clone + Send + Sync> ComponentS component_id: &ComponentId, component_name: &ComponentName, _data: Vec, - _namespace: &Namespace, - ) -> Result { + namespace: &Namespace, + ) -> Result, ComponentError> { let fake_component = Component { + namespace: namespace.clone(), component_name: component_name.clone(), component_size: 0, metadata: ComponentMetadata { @@ -663,9 +676,10 @@ impl + Eq + Clone + Send + Sync> ComponentS &self, component_id: &ComponentId, _data: Vec, - _namespace: &Namespace, - ) -> Result { + namespace: &Namespace, + ) -> Result, ComponentError> { let fake_component = Component { + namespace: namespace.clone(), component_name: ComponentName("fake".to_string()), component_size: 0, metadata: ComponentMetadata { @@ -733,7 +747,7 @@ impl + Eq + Clone + Send + Sync> ComponentS &self, _component_name: Option, _namespace: &Namespace, - ) -> Result, ComponentError> { + ) -> Result>, ComponentError> { Ok(vec![]) } @@ -741,7 +755,7 @@ impl + Eq + Clone + Send + Sync> ComponentS &self, _component_id: &VersionedComponentId, _namespace: &Namespace, - ) -> Result, ComponentError> { + ) -> Result>, ComponentError> { Ok(None) } @@ -749,7 +763,7 @@ impl + Eq + Clone + Send + Sync> ComponentS &self, _component_id: &ComponentId, _namespace: &Namespace, - ) -> Result, ComponentError> { + ) -> Result>, ComponentError> { Ok(None) } @@ -757,7 +771,7 @@ impl + Eq + Clone + Send + Sync> ComponentS &self, _component_id: &ComponentId, _namespace: &Namespace, - ) -> Result, ComponentError> { + ) -> Result>, ComponentError> { Ok(vec![]) } diff --git a/golem-component-service-base/tests/services_tests.rs b/golem-component-service-base/tests/services_tests.rs index acdb6dff80..190e1452d3 100644 --- a/golem-component-service-base/tests/services_tests.rs +++ b/golem-component-service-base/tests/services_tests.rs @@ -5,9 +5,8 @@ mod tests { use golem_service_base::db; use golem_common::model::ComponentId; - use golem_component_service_base::repo::component::{ - ComponentRecord, ComponentRepo, DbComponentRepo, - }; + use golem_component_service_base::model::Component; + use golem_component_service_base::repo::component::{ComponentRepo, DbComponentRepo}; use golem_component_service_base::service::component::{ create_new_component, ComponentService, ComponentServiceDefault, }; @@ -295,19 +294,24 @@ mod tests { let data = get_component_data("shopping-cart"); let component1 = - create_new_component(&ComponentId::new_v4(), &component_name1, &data).unwrap(); + create_new_component(&ComponentId::new_v4(), &component_name1, &data, &namespace1) + .unwrap(); let result1 = component_repo - .create(&ComponentRecord::new(namespace1.clone(), component1.clone()).unwrap()) + .create(&component1.clone().try_into().unwrap()) .await; let result2 = component_repo - .create( - &ComponentRecord::new(namespace1.clone(), component1.clone().next_version()) - .unwrap(), - ) + .create(&component1.clone().next_version().try_into().unwrap()) .await; let result3 = component_repo - .create(&ComponentRecord::new(namespace2.clone(), component1.clone()).unwrap()) + .create( + &Component { + namespace: namespace2.clone(), + ..component1.clone() + } + .try_into() + .unwrap(), + ) .await; assert!(result1.is_ok()); @@ -325,18 +329,27 @@ mod tests { let data = get_component_data("shopping-cart"); let component1 = - create_new_component(&ComponentId::new_v4(), &component_name1, &data).unwrap(); + create_new_component(&ComponentId::new_v4(), &component_name1, &data, &namespace1) + .unwrap(); let component2 = - create_new_component(&ComponentId::new_v4(), &component_name1, &data).unwrap(); + create_new_component(&ComponentId::new_v4(), &component_name1, &data, &namespace2) + .unwrap(); let result1 = component_repo - .create(&ComponentRecord::new(namespace1.clone(), component1.clone()).unwrap()) + .create(&component1.clone().try_into().unwrap()) .await; let result2 = component_repo - .create(&ComponentRecord::new(namespace1.clone(), component2.clone()).unwrap()) + .create( + &Component { + namespace: namespace2.clone(), + ..component1.clone() + } + .try_into() + .unwrap(), + ) .await; let result3 = component_repo - .create(&ComponentRecord::new(namespace2.clone(), component2.clone()).unwrap()) + .create(&component2.clone().try_into().unwrap()) .await; assert!(result1.is_ok()); @@ -351,10 +364,11 @@ mod tests { let data = get_component_data("shopping-cart"); let component1 = - create_new_component(&ComponentId::new_v4(), &component_name1, &data).unwrap(); + create_new_component(&ComponentId::new_v4(), &component_name1, &data, &namespace1) + .unwrap(); let result1 = component_repo - .create(&ComponentRecord::new(namespace1.clone(), component1.clone()).unwrap()) + .create(&component1.clone().try_into().unwrap()) .await; let result2 = component_repo diff --git a/golem-component-service/src/api/component.rs b/golem-component-service/src/api/component.rs index 3d5b68be5c..ecce357490 100644 --- a/golem-component-service/src/api/component.rs +++ b/golem-component-service/src/api/component.rs @@ -116,7 +116,7 @@ impl ComponentApi { &DefaultNamespace::default(), ) .await?; - Ok(Json(response)) + Ok(Json(response.into())) } #[oai( @@ -134,7 +134,7 @@ impl ComponentApi { .component_service .update(&component_id.0, data, &DefaultNamespace::default()) .await?; - Ok(Json(response)) + Ok(Json(response.into())) } #[oai( @@ -169,7 +169,7 @@ impl ComponentApi { .component_service .get(&component_id.0, &DefaultNamespace::default()) .await?; - Ok(Json(response)) + Ok(Json(response.into_iter().map(|c| c.into()).collect())) } #[oai( @@ -202,7 +202,7 @@ impl ComponentApi { .await?; match response { - Some(component) => Ok(Json(component)), + Some(component) => Ok(Json(component.into())), None => Err(ComponentError::NotFound(Json(ErrorBody { error: "Component not found".to_string(), }))), @@ -224,7 +224,7 @@ impl ComponentApi { .await?; match response { - Some(component) => Ok(Json(component)), + Some(component) => Ok(Json(component.into())), None => Err(ComponentError::NotFound(Json(ErrorBody { error: "Component not found".to_string(), }))), @@ -241,6 +241,6 @@ impl ComponentApi { .find_by_name(component_name.0, &DefaultNamespace::default()) .await?; - Ok(Json(response)) + Ok(Json(response.into_iter().map(|c| c.into()).collect())) } } From 803db08ef086f28017a2379bd0adf14832e4964f Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Tue, 9 Jul 2024 13:52:07 +0200 Subject: [PATCH 27/30] repo - select by id without namespace --- .../src/repo/component.rs | 42 +--- .../src/service/component.rs | 203 +++++++++--------- .../tests/services_tests.rs | 57 ++++- .../src/service/component_object_store.rs | 45 +++- 4 files changed, 204 insertions(+), 143 deletions(-) diff --git a/golem-component-service-base/src/repo/component.rs b/golem-component-service-base/src/repo/component.rs index dffb7c3d06..e5877b8f99 100644 --- a/golem-component-service-base/src/repo/component.rs +++ b/golem-component-service-base/src/repo/component.rs @@ -101,23 +101,17 @@ where pub trait ComponentRepo { async fn create(&self, component: &ComponentRecord) -> Result<(), RepoError>; - async fn get( - &self, - namespace: &str, - component_id: &Uuid, - ) -> Result, RepoError>; + async fn get(&self, component_id: &Uuid) -> Result, RepoError>; async fn get_all(&self, namespace: &str) -> Result, RepoError>; async fn get_latest_version( &self, - namespace: &str, component_id: &Uuid, ) -> Result, RepoError>; async fn get_by_version( &self, - namespace: &str, component_id: &Uuid, version: u64, ) -> Result, RepoError>; @@ -198,11 +192,7 @@ impl ComponentRepo for DbComponentRepo { Ok(()) } - async fn get( - &self, - namespace: &str, - component_id: &Uuid, - ) -> Result, RepoError> { + async fn get(&self, component_id: &Uuid) -> Result, RepoError> { sqlx::query_as::<_, ComponentRecord>( r#" SELECT @@ -214,10 +204,9 @@ impl ComponentRepo for DbComponentRepo { cv.metadata AS metadata FROM components c JOIN component_versions cv ON c.component_id = cv.component_id - WHERE c.namespace = $1 AND c.component_id = $2 + WHERE c.component_id = $1 "#, ) - .bind(namespace) .bind(component_id) .fetch_all(self.db_pool.deref()) .await @@ -247,7 +236,6 @@ impl ComponentRepo for DbComponentRepo { async fn get_latest_version( &self, - namespace: &str, component_id: &Uuid, ) -> Result, RepoError> { sqlx::query_as::<_, ComponentRecord>( @@ -261,11 +249,10 @@ impl ComponentRepo for DbComponentRepo { cv.metadata AS metadata FROM components c JOIN component_versions cv ON c.component_id = cv.component_id - WHERE c.namespace = $1 AND c.component_id = $2 + WHERE c.component_id = $1 ORDER BY cv.version DESC LIMIT 1 "#, ) - .bind(namespace) .bind(component_id) .fetch_optional(self.db_pool.deref()) .await @@ -274,7 +261,6 @@ impl ComponentRepo for DbComponentRepo { async fn get_by_version( &self, - namespace: &str, component_id: &Uuid, version: u64, ) -> Result, RepoError> { @@ -289,10 +275,9 @@ impl ComponentRepo for DbComponentRepo { cv.metadata AS metadata FROM components c JOIN component_versions cv ON c.component_id = cv.component_id - WHERE c.namespace = $1 AND c.component_id = $2 AND cv.version = $3 + WHERE c.component_id = $1 AND cv.version = $2 "#, ) - .bind(namespace) .bind(component_id) .bind(version as i64) .fetch_optional(self.db_pool.deref()) @@ -423,11 +408,7 @@ impl ComponentRepo for DbComponentRepo { Ok(()) } - async fn get( - &self, - namespace: &str, - component_id: &Uuid, - ) -> Result, RepoError> { + async fn get(&self, component_id: &Uuid) -> Result, RepoError> { sqlx::query_as::<_, ComponentRecord>( r#" SELECT @@ -439,10 +420,9 @@ impl ComponentRepo for DbComponentRepo { cv.metadata AS metadata FROM components c JOIN component_versions cv ON c.component_id = cv.component_id - WHERE c.namespace = $1 AND c.component_id = $2 + WHERE c.component_id = $1 "#, ) - .bind(namespace) .bind(component_id) .fetch_all(self.db_pool.deref()) .await @@ -472,7 +452,6 @@ impl ComponentRepo for DbComponentRepo { async fn get_latest_version( &self, - namespace: &str, component_id: &Uuid, ) -> Result, RepoError> { sqlx::query_as::<_, ComponentRecord>( @@ -486,11 +465,10 @@ impl ComponentRepo for DbComponentRepo { cv.metadata AS metadata FROM components c JOIN component_versions cv ON c.component_id = cv.component_id - WHERE c.namespace = $1 AND c.component_id = $2 + WHERE c.component_id = $1 ORDER BY cv.version DESC LIMIT 1 "#, ) - .bind(namespace) .bind(component_id) .fetch_optional(self.db_pool.deref()) .await @@ -499,7 +477,6 @@ impl ComponentRepo for DbComponentRepo { async fn get_by_version( &self, - namespace: &str, component_id: &Uuid, version: u64, ) -> Result, RepoError> { @@ -514,10 +491,9 @@ impl ComponentRepo for DbComponentRepo { cv.metadata AS metadata FROM components c JOIN component_versions cv ON c.component_id = cv.component_id - WHERE c.namespace = $1 AND c.component_id = $2 AND cv.version = $3 + WHERE c.component_id = $1 AND cv.version = $2 "#, ) - .bind(namespace) .bind(component_id) .bind(version as i64) .fetch_optional(self.db_pool.deref()) diff --git a/golem-component-service-base/src/service/component.rs b/golem-component-service-base/src/service/component.rs index ea68a422a7..73ff280803 100644 --- a/golem-component-service-base/src/service/component.rs +++ b/golem-component-service-base/src/service/component.rs @@ -167,6 +167,12 @@ pub trait ComponentService { &self, component_id: &ComponentId, ) -> Result, ComponentError>; + + async fn delete( + &self, + component_id: &ComponentId, + namespace: &Namespace, + ) -> Result<(), ComponentError>; } pub struct ComponentServiceDefault { @@ -253,8 +259,9 @@ where let next_component = self .component_repo - .get_latest_version(namespace.to_string().as_str(), &component_id.0) + .get_latest_version(&component_id.0) .await? + .filter(|c| c.namespace == namespace.to_string()) .ok_or(ComponentError::UnknownComponentId(component_id.clone())) .and_then(|c| { c.try_into() @@ -305,29 +312,11 @@ where version: Option, namespace: &Namespace, ) -> Result, ComponentError> { - let versioned_component_id = { - match version { - Some(version) => { - let stored_namespace: Option = - self.get_namespace(component_id).await?; - match stored_namespace { - Some(stored_namespace) if stored_namespace == namespace.clone() => { - VersionedComponentId { - component_id: component_id.clone(), - version, - } - } - _ => return Err(ComponentError::UnknownComponentId(component_id.clone())), - } - } - None => self - .component_repo - .get_latest_version(namespace.to_string().as_str(), &component_id.0) - .await? - .map(|c| c.into()) - .ok_or(ComponentError::UnknownComponentId(component_id.clone()))?, - } - }; + let versioned_component_id = self + .get_versioned_component_id(component_id, version, namespace) + .await? + .ok_or(ComponentError::UnknownComponentId(component_id.clone()))?; + info!( "Downloading component - namespace: {}, id: {}, version: {}", namespace, component_id, versioned_component_id.version @@ -350,29 +339,11 @@ where version: Option, namespace: &Namespace, ) -> Result { - let versioned_component_id = { - match version { - Some(version) => { - let stored_namespace: Option = - self.get_namespace(component_id).await?; - match stored_namespace { - Some(stored_namespace) if stored_namespace == namespace.clone() => { - VersionedComponentId { - component_id: component_id.clone(), - version, - } - } - _ => return Err(ComponentError::UnknownComponentId(component_id.clone())), - } - } - None => self - .component_repo - .get_latest_version(namespace.to_string().as_str(), &component_id.0) - .await? - .map(|c| c.into()) - .ok_or(ComponentError::UnknownComponentId(component_id.clone()))?, - } - }; + let versioned_component_id = self + .get_versioned_component_id(component_id, version, namespace) + .await? + .ok_or(ComponentError::UnknownComponentId(component_id.clone()))?; + info!( "Downloading component - namespace: {}, id: {}, version: {}", namespace, component_id, versioned_component_id.version @@ -402,43 +373,27 @@ where version.map_or("N/A".to_string(), |v| v.to_string()) ); - let latest_component = self - .component_repo - .get_latest_version(namespace.to_string().as_str(), &component_id.0) + let versioned_component_id = self + .get_versioned_component_id(component_id, version, namespace) .await?; - let v = match latest_component { - Some(component) => match version { - Some(v) if v <= component.version as u64 => v, - None => component.version as u64, - _ => { - return Ok(None); - } - }, - None => { - return Ok(None); + match versioned_component_id { + Some(versioned_component_id) => { + let id = ProtectedComponentId { + versioned_component_id, + }; + let data = self + .object_store + .get(&self.get_protected_object_store_key(&id)) + .await + .tap_err(|e| error!("Error retrieving component: {}", e)) + .map_err(|e| { + ComponentError::internal(e.to_string(), "Error retrieving component") + })?; + Ok(Some(data)) } - }; - - let versioned_component_id = VersionedComponentId { - component_id: component_id.clone(), - version: v, - }; - - let protected_id = ProtectedComponentId { - versioned_component_id, - }; - - let object_key = self.get_protected_object_store_key(&protected_id); - - let result = self - .object_store - .get(&object_key) - .await - .tap_err(|e| error!("Error retrieving component: {}", e)) - .map_err(|e| ComponentError::internal(e.to_string(), "Error retrieving component"))?; - - Ok(Some(result)) + None => Ok(None), + } } async fn find_id_by_name( @@ -479,7 +434,7 @@ where let values: Vec> = records .iter() - .map(|d| d.clone().try_into()) + .map(|c| c.clone().try_into()) .collect::>, _>>() .map_err(|e| ComponentError::internal(e, "Failed to convert record".to_string()))?; @@ -495,14 +450,12 @@ where "Getting component - namespace: {}, id: {}", namespace, component_id ); - let records = self - .component_repo - .get(namespace.to_string().as_str(), &component_id.0) - .await?; + let records = self.component_repo.get(&component_id.0).await?; let values: Vec> = records .iter() - .map(|d| d.clone().try_into()) + .filter(|d| d.namespace == namespace.to_string()) + .map(|c| c.clone().try_into()) .collect::>, _>>() .map_err(|e| ComponentError::internal(e, "Failed to convert record".to_string()))?; @@ -521,21 +474,17 @@ where let result = self .component_repo - .get_by_version( - namespace.to_string().as_str(), - &component_id.component_id.0, - component_id.version, - ) + .get_by_version(&component_id.component_id.0, component_id.version) .await?; match result { - Some(c) => { + Some(c) if c.namespace == namespace.to_string() => { let value = c.try_into().map_err(|e| { ComponentError::internal(e, "Failed to convert record".to_string()) })?; Ok(Some(value)) } - None => Ok(None), + _ => Ok(None), } } @@ -550,17 +499,17 @@ where ); let result = self .component_repo - .get_latest_version(namespace.to_string().as_str(), &component_id.0) + .get_latest_version(&component_id.0) .await?; match result { - Some(c) => { + Some(c) if c.namespace == namespace.to_string() => { let value = c.try_into().map_err(|e| { ComponentError::internal(e, "Failed to convert record".to_string()) })?; Ok(Some(value)) } - None => Ok(None), + _ => Ok(None), } } @@ -579,6 +528,29 @@ where Ok(None) } } + + async fn delete( + &self, + component_id: &ComponentId, + namespace: &Namespace, + ) -> Result<(), ComponentError> { + info!( + "Deleting component - namespace: {}, id: {}, version: latest", + namespace, component_id + ); + let versioned_component_id = self + .get_versioned_component_id(component_id, None, namespace) + .await?; + if versioned_component_id.is_some() { + // TODO delete from object store + self.component_repo + .delete(namespace.to_string().as_str(), &component_id.0) + .await?; + Ok(()) + } else { + Err(ComponentError::UnknownComponentId(component_id.clone())) + } + } } impl ComponentServiceDefault { @@ -626,6 +598,35 @@ impl ComponentServiceDefault { ComponentError::internal(e.to_string(), "Failed to upload protected component") }) } + + async fn get_versioned_component_id( + &self, + component_id: &ComponentId, + version: Option, + namespace: &Namespace, + ) -> Result, ComponentError> { + let stored = self + .component_repo + .get_latest_version(&component_id.0) + .await?; + + match stored { + Some(stored) if stored.namespace == namespace.to_string() => { + let stored_version = stored.version as u64; + let requested_version = version.unwrap_or(stored_version); + + if requested_version <= stored_version { + Ok(Some(VersionedComponentId { + component_id: component_id.clone(), + version: requested_version, + })) + } else { + Ok(None) + } + } + _ => Ok(None), + } + } } #[derive(Default)] @@ -781,4 +782,12 @@ impl + Eq + Clone + Send + Sync> ComponentS ) -> Result, ComponentError> { Ok(None) } + + async fn delete( + &self, + _component_id: &ComponentId, + _namespace: &Namespace, + ) -> Result<(), ComponentError> { + Ok(()) + } } diff --git a/golem-component-service-base/tests/services_tests.rs b/golem-component-service-base/tests/services_tests.rs index 190e1452d3..2b5109b3c3 100644 --- a/golem-component-service-base/tests/services_tests.rs +++ b/golem-component-service-base/tests/services_tests.rs @@ -243,6 +243,36 @@ mod tests { .unwrap(); assert!(!component2_result.is_empty()); + let component1_result = component_service + .get_protected_data( + &component1v2.versioned_component_id.component_id, + Some(component1v2.versioned_component_id.version), + &DefaultNamespace::default(), + ) + .await + .unwrap(); + assert!(component1_result.is_some()); + + let component1_result = component_service + .get_protected_data( + &component1v2.versioned_component_id.component_id, + Some(10000000), + &DefaultNamespace::default(), + ) + .await + .unwrap(); + assert!(component1_result.is_none()); + + let component2_result = component_service + .get_protected_data( + &component1v2.versioned_component_id.component_id, + None, + &DefaultNamespace::default(), + ) + .await + .unwrap(); + assert!(component2_result.is_some()); + let component1_result = component_service .find_id_by_name(&component1.component_name, &DefaultNamespace::default()) .await @@ -278,6 +308,23 @@ mod tests { .await .unwrap(); assert!(component_result.len() == 3); + + component_service + .delete( + &component1v2.versioned_component_id.component_id, + &DefaultNamespace::default(), + ) + .await + .unwrap(); + + let component1_result = component_service + .get( + &component1.versioned_component_id.component_id, + &DefaultNamespace::default(), + ) + .await + .unwrap(); + assert!(component1_result.is_empty()); } async fn test_repo(component_repo: Arc) { @@ -372,10 +419,7 @@ mod tests { .await; let result2 = component_repo - .get( - &namespace1, - &component1.versioned_component_id.component_id.0, - ) + .get(&component1.versioned_component_id.component_id.0) .await; let result3 = component_repo @@ -386,10 +430,7 @@ mod tests { .await; let result4 = component_repo - .get( - &namespace1, - &component1.versioned_component_id.component_id.0, - ) + .get(&component1.versioned_component_id.component_id.0) .await; assert!(result1.is_ok()); diff --git a/golem-service-base/src/service/component_object_store.rs b/golem-service-base/src/service/component_object_store.rs index 48018ed58b..0d38e5b19e 100644 --- a/golem-service-base/src/service/component_object_store.rs +++ b/golem-service-base/src/service/component_object_store.rs @@ -12,16 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::fs; -use std::path::PathBuf; -use std::pin::Pin; -use std::task::{Context, Poll}; - use crate::config::{ComponentStoreLocalConfig, ComponentStoreS3Config}; use crate::stream::ByteStream; +use anyhow::Error; use async_trait::async_trait; use aws_config::BehaviorVersion; use futures::Stream; +use std::fs; +use std::path::PathBuf; +use std::pin::Pin; +use std::task::{Context, Poll}; use tracing::{debug, info}; #[async_trait] @@ -31,6 +31,8 @@ pub trait ComponentObjectStore { async fn get_stream(&self, object_key: &str) -> ByteStream; async fn put(&self, object_key: &str, data: Vec) -> Result<(), anyhow::Error>; + + async fn delete(&self, object_key: &str) -> Result<(), anyhow::Error>; } pub struct AwsByteStream(aws_sdk_s3::primitives::ByteStream); @@ -133,6 +135,21 @@ impl ComponentObjectStore for AwsS3ComponentObjectStore { Ok(()) } + + async fn delete(&self, object_key: &str) -> Result<(), anyhow::Error> { + let key = self.get_key(object_key); + + info!("Deleting object: {}/{}", self.bucket_name, key); + + self.client + .delete_object() + .bucket(&self.bucket_name) + .key(key) + .send() + .await?; + + Ok(()) + } } pub struct FsComponentObjectStore { @@ -210,6 +227,24 @@ impl ComponentObjectStore for FsComponentObjectStore { fs::write(file_path, data).map_err(|e| e.into()) } + + async fn delete(&self, object_key: &str) -> Result<(), Error> { + let dir_path = self.get_dir_path(); + + debug!("Deleting object: {}/{}", dir_path.display(), object_key); + + if !dir_path.exists() { + fs::create_dir_all(dir_path.clone())?; + } + + let file_path = dir_path.join(object_key); + + if file_path.exists() { + fs::remove_file(file_path)?; + } + + Ok(()) + } } #[cfg(test)] From 8ed30a6c1a2c0c8d13c70acad20ab3367de27bd0 Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Tue, 9 Jul 2024 19:11:31 +0200 Subject: [PATCH 28/30] component repo - delete --- .../src/service/component.rs | 35 +++++- .../src/repo/api_definition.rs | 115 +----------------- 2 files changed, 30 insertions(+), 120 deletions(-) diff --git a/golem-component-service-base/src/service/component.rs b/golem-component-service-base/src/service/component.rs index 73ff280803..6681de6a31 100644 --- a/golem-component-service-base/src/service/component.rs +++ b/golem-component-service-base/src/service/component.rs @@ -535,14 +535,37 @@ where namespace: &Namespace, ) -> Result<(), ComponentError> { info!( - "Deleting component - namespace: {}, id: {}, version: latest", + "Deleting component - namespace: {}, id: {}", namespace, component_id ); - let versioned_component_id = self - .get_versioned_component_id(component_id, None, namespace) - .await?; - if versioned_component_id.is_some() { - // TODO delete from object store + + let records = self.component_repo.get(&component_id.0).await?; + + let versioned_component_ids: Vec = records + .into_iter() + .filter(|d| d.namespace == namespace.to_string()) + .map(|c| c.into()) + .collect(); + + if !versioned_component_ids.is_empty() { + for versioned_component_id in versioned_component_ids { + self.object_store + .delete(&self.get_protected_object_store_key(&ProtectedComponentId { + versioned_component_id: versioned_component_id.clone(), + })) + .await + .map_err(|e| { + ComponentError::internal(e.to_string(), "Failed to delete component") + })?; + self.object_store + .delete(&self.get_user_object_store_key(&UserComponentId { + versioned_component_id: versioned_component_id.clone(), + })) + .await + .map_err(|e| { + ComponentError::internal(e.to_string(), "Failed to delete component") + })?; + } self.component_repo .delete(namespace.to_string().as_str(), &component_id.0) .await?; diff --git a/golem-worker-service-base/src/repo/api_definition.rs b/golem-worker-service-base/src/repo/api_definition.rs index 1647d6c051..ab4154e08f 100644 --- a/golem-worker-service-base/src/repo/api_definition.rs +++ b/golem-worker-service-base/src/repo/api_definition.rs @@ -16,10 +16,9 @@ use crate::api_definition::http::HttpApiDefinition; use crate::repo::RepoError; use async_trait::async_trait; use sqlx::{Database, Pool, Row}; -use std::collections::HashMap; use std::fmt::Display; use std::ops::Deref; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; #[derive(sqlx::FromRow, Debug, Clone)] pub struct ApiDefinitionRecord { @@ -374,118 +373,6 @@ impl ApiDefinitionRepo for DbApiDefinitionRepo { } } -pub struct InMemoryApiDefinitionRepo { - registry: Mutex>, -} - -impl Default for InMemoryApiDefinitionRepo { - fn default() -> Self { - Self { - registry: Mutex::new(HashMap::new()), - } - } -} - -#[async_trait] -impl ApiDefinitionRepo for InMemoryApiDefinitionRepo { - async fn create(&self, definition: &ApiDefinitionRecord) -> Result<(), RepoError> { - let key = ( - definition.namespace.clone(), - definition.id.clone(), - definition.version.clone(), - ); - let mut registry = self.registry.lock().unwrap(); - if let std::collections::hash_map::Entry::Vacant(e) = registry.entry(key.clone()) { - e.insert(definition.clone()); - Ok(()) - } else { - Err(RepoError::Internal( - "ApiDefinition already exists".to_string(), - )) - } - } - - async fn update(&self, definition: &ApiDefinitionRecord) -> Result<(), RepoError> { - let key = ( - definition.namespace.clone(), - definition.id.clone(), - definition.version.clone(), - ); - let mut registry = self.registry.lock().unwrap(); - registry.insert(key.clone(), definition.clone()); - Ok(()) - } - - async fn set_not_draft( - &self, - namespace: &str, - id: &str, - version: &str, - ) -> Result<(), RepoError> { - match self.get(namespace, id, version).await? { - Some(v) if v.draft => { - let mut registry = self.registry.lock().unwrap(); - let key = (namespace.to_string(), id.to_string(), version.to_string()); - registry.entry(key.clone()).and_modify(|v| v.draft = false); - Ok(()) - } - _ => Ok(()), - } - } - - async fn get( - &self, - namespace: &str, - id: &str, - version: &str, - ) -> Result, RepoError> { - let key = (namespace.to_string(), id.to_string(), version.to_string()); - let registry = self.registry.lock().unwrap(); - Ok(registry.get(&key).cloned()) - } - - async fn get_draft( - &self, - namespace: &str, - id: &str, - version: &str, - ) -> Result, RepoError> { - let value = self.get(namespace, id, version).await?; - Ok(value.map(|v| v.draft)) - } - - async fn delete(&self, namespace: &str, id: &str, version: &str) -> Result { - let key = (namespace.to_string(), id.to_string(), version.to_string()); - let mut registry = self.registry.lock().unwrap(); - let result = registry.remove(&key); - Ok(result.is_some()) - } - - async fn get_all(&self, namespace: &str) -> Result, RepoError> { - let registry = self.registry.lock().unwrap(); - let result: Vec = registry - .iter() - .filter(|(k, _)| k.0 == *namespace) - .map(|(_, v)| v.clone()) - .collect(); - Ok(result) - } - - async fn get_all_versions( - &self, - namespace: &str, - id: &str, - ) -> Result, RepoError> { - let registry = self.registry.lock().unwrap(); - let result = registry - .iter() - .filter(|(k, _)| k.0 == *namespace && k.1 == *id) - .map(|(_, v)| v.clone()) - .collect(); - Ok(result) - } -} - pub mod record_data_serde { use crate::api_definition::http::Route; use bytes::{BufMut, Bytes, BytesMut}; From eb396e0b078177bdaba816eadf47cc856ffe1658 Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Tue, 9 Jul 2024 23:41:53 +0200 Subject: [PATCH 29/30] logs, tests --- .../src/service/component.rs | 25 +++++++++++++++---- .../tests/services_tests.rs | 10 ++++++++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/golem-component-service-base/src/service/component.rs b/golem-component-service-base/src/service/component.rs index 6681de6a31..6b542e038c 100644 --- a/golem-component-service-base/src/service/component.rs +++ b/golem-component-service-base/src/service/component.rs @@ -323,13 +323,21 @@ where ); let id = ProtectedComponentId { - versioned_component_id, + versioned_component_id: versioned_component_id.clone(), }; self.object_store .get(&self.get_protected_object_store_key(&id)) .await - .tap_err(|e| error!("Error downloading component: {}", e)) + .tap_err(|e| { + error!( + "Error downloading component - namespace: {}, id: {}, version: {}, error: {}", + namespace, + versioned_component_id.component_id, + versioned_component_id.version, + e + ) + }) .map_err(|e| ComponentError::internal(e.to_string(), "Error downloading component")) } @@ -380,13 +388,20 @@ where match versioned_component_id { Some(versioned_component_id) => { let id = ProtectedComponentId { - versioned_component_id, + versioned_component_id: versioned_component_id.clone(), }; let data = self .object_store .get(&self.get_protected_object_store_key(&id)) .await - .tap_err(|e| error!("Error retrieving component: {}", e)) + .tap_err(|e| { + error!("Error getting component data - namespace: {}, id: {}, version: {}, error: {}", + namespace, + versioned_component_id.component_id, + versioned_component_id.version, + e + ) + }) .map_err(|e| { ComponentError::internal(e.to_string(), "Error retrieving component") })?; @@ -656,7 +671,7 @@ impl ComponentServiceDefault { pub struct ComponentServiceNoop {} #[async_trait] -impl + Eq + Clone + Send + Sync> ComponentService +impl ComponentService for ComponentServiceNoop { async fn create( diff --git a/golem-component-service-base/tests/services_tests.rs b/golem-component-service-base/tests/services_tests.rs index 2b5109b3c3..523d0648fa 100644 --- a/golem-component-service-base/tests/services_tests.rs +++ b/golem-component-service-base/tests/services_tests.rs @@ -325,6 +325,16 @@ mod tests { .await .unwrap(); assert!(component1_result.is_empty()); + + let component1_result = component_service + .get_protected_data( + &component1v2.versioned_component_id.component_id, + Some(component1v2.versioned_component_id.version), + &DefaultNamespace::default(), + ) + .await + .unwrap(); + assert!(component1_result.is_none()); } async fn test_repo(component_repo: Arc) { From a75cc52228f8ff37ed4cce5d5407f47beaf6eceb Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Wed, 10 Jul 2024 14:08:00 +0200 Subject: [PATCH 30/30] fixes --- .../src/repo/component.rs | 14 ++++++----- .../tests/db/migration/postgres/001__init.sql | 20 ---------------- .../tests/db/migration/sqlite/001__init.sql | 20 ---------------- .../tests/services_tests.rs | 11 +++++---- .../db/migration/postgres/001__init.sql | 2 -- .../db/migration/sqlite/001__init.sql | 2 -- .../tests/db/migration/postgres/001__init.sql | 24 ------------------- .../tests/db/migration/sqlite/001__init.sql | 23 ------------------ .../tests/services_tests.rs | 4 ++-- 9 files changed, 17 insertions(+), 103 deletions(-) delete mode 100644 golem-component-service-base/tests/db/migration/postgres/001__init.sql delete mode 100644 golem-component-service-base/tests/db/migration/sqlite/001__init.sql delete mode 100644 golem-worker-service-base/tests/db/migration/postgres/001__init.sql delete mode 100644 golem-worker-service-base/tests/db/migration/sqlite/001__init.sql diff --git a/golem-component-service-base/src/repo/component.rs b/golem-component-service-base/src/repo/component.rs index e5877b8f99..d3a0c4b6bb 100644 --- a/golem-component-service-base/src/repo/component.rs +++ b/golem-component-service-base/src/repo/component.rs @@ -144,16 +144,17 @@ impl ComponentRepo for DbComponentRepo { async fn create(&self, component: &ComponentRecord) -> Result<(), RepoError> { let mut transaction = self.db_pool.begin().await?; - let result = sqlx::query("SELECT namespace FROM components WHERE component_id = $1") + let result = sqlx::query("SELECT namespace, name FROM components WHERE component_id = $1") .bind(component.component_id) .fetch_optional(&mut *transaction) .await?; if let Some(result) = result { let namespace: String = result.get("namespace"); - if namespace != component.namespace { + let name: String = result.get("name"); + if namespace != component.namespace || name != component.name { return Err(RepoError::Internal( - "Component namespace invalid".to_string(), + "Component namespace and name invalid".to_string(), )); } } else { @@ -360,16 +361,17 @@ impl ComponentRepo for DbComponentRepo { async fn create(&self, component: &ComponentRecord) -> Result<(), RepoError> { let mut transaction = self.db_pool.begin().await?; - let result = sqlx::query("SELECT namespace FROM components WHERE component_id = $1") + let result = sqlx::query("SELECT namespace, name FROM components WHERE component_id = $1") .bind(component.component_id) .fetch_optional(&mut *transaction) .await?; if let Some(result) = result { let namespace: String = result.get("namespace"); - if namespace != component.namespace { + let name: String = result.get("name"); + if namespace != component.namespace || name != component.name { return Err(RepoError::Internal( - "Component namespace invalid".to_string(), + "Component namespace and name invalid".to_string(), )); } } else { diff --git a/golem-component-service-base/tests/db/migration/postgres/001__init.sql b/golem-component-service-base/tests/db/migration/postgres/001__init.sql deleted file mode 100644 index 66d60275e1..0000000000 --- a/golem-component-service-base/tests/db/migration/postgres/001__init.sql +++ /dev/null @@ -1,20 +0,0 @@ -CREATE TABLE components -( - component_id uuid NOT NULL PRIMARY KEY, - namespace text NOT NULL, - name text NOT NULL -); - -CREATE INDEX components_namespace_id_idx ON components (namespace, component_id); - -CREATE UNIQUE INDEX components_namespace_name_idx ON components (namespace, name); - -CREATE TABLE component_versions -( - component_id uuid NOT NULL REFERENCES components (component_id), - version bigint NOT NULL, - size integer NOT NULL, - created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - metadata bytea NOT NULL, - PRIMARY KEY (component_id, version) -); diff --git a/golem-component-service-base/tests/db/migration/sqlite/001__init.sql b/golem-component-service-base/tests/db/migration/sqlite/001__init.sql deleted file mode 100644 index fa088d860e..0000000000 --- a/golem-component-service-base/tests/db/migration/sqlite/001__init.sql +++ /dev/null @@ -1,20 +0,0 @@ -CREATE TABLE components -( - component_id uuid NOT NULL PRIMARY KEY, - namespace text NOT NULL, - name text NOT NULL -); - -CREATE INDEX components_namespace_id_idx ON components (namespace, component_id); - -CREATE UNIQUE INDEX components_namespace_name_idx ON components (namespace, name); - -CREATE TABLE component_versions -( - component_id uuid NOT NULL REFERENCES components (component_id), - version bigint NOT NULL, - size integer NOT NULL, - created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, - metadata blob NOT NULL, - PRIMARY KEY (component_id, version) -); diff --git a/golem-component-service-base/tests/services_tests.rs b/golem-component-service-base/tests/services_tests.rs index 523d0648fa..43252f76a8 100644 --- a/golem-component-service-base/tests/services_tests.rs +++ b/golem-component-service-base/tests/services_tests.rs @@ -61,9 +61,12 @@ mod tests { let cli = Cli::default(); let (db_config, _container) = start_docker_postgres(&cli); - db::postgres_migrate(&db_config, "tests/db/migration/postgres") - .await - .unwrap(); + db::postgres_migrate( + &db_config, + "../golem-component-service/db/migration/postgres", + ) + .await + .unwrap(); let db_pool = db::create_postgres_pool(&db_config).await.unwrap(); @@ -82,7 +85,7 @@ mod tests { max_connections: 10, }; - db::sqlite_migrate(&db_config, "tests/db/migration/sqlite") + db::sqlite_migrate(&db_config, "../golem-component-service/db/migration/sqlite") .await .unwrap(); diff --git a/golem-component-service/db/migration/postgres/001__init.sql b/golem-component-service/db/migration/postgres/001__init.sql index 66d60275e1..7bc44fbec2 100644 --- a/golem-component-service/db/migration/postgres/001__init.sql +++ b/golem-component-service/db/migration/postgres/001__init.sql @@ -5,8 +5,6 @@ CREATE TABLE components name text NOT NULL ); -CREATE INDEX components_namespace_id_idx ON components (namespace, component_id); - CREATE UNIQUE INDEX components_namespace_name_idx ON components (namespace, name); CREATE TABLE component_versions diff --git a/golem-component-service/db/migration/sqlite/001__init.sql b/golem-component-service/db/migration/sqlite/001__init.sql index fa088d860e..79ea965148 100644 --- a/golem-component-service/db/migration/sqlite/001__init.sql +++ b/golem-component-service/db/migration/sqlite/001__init.sql @@ -5,8 +5,6 @@ CREATE TABLE components name text NOT NULL ); -CREATE INDEX components_namespace_id_idx ON components (namespace, component_id); - CREATE UNIQUE INDEX components_namespace_name_idx ON components (namespace, name); CREATE TABLE component_versions diff --git a/golem-worker-service-base/tests/db/migration/postgres/001__init.sql b/golem-worker-service-base/tests/db/migration/postgres/001__init.sql deleted file mode 100644 index dc71829400..0000000000 --- a/golem-worker-service-base/tests/db/migration/postgres/001__init.sql +++ /dev/null @@ -1,24 +0,0 @@ -CREATE TABLE api_definitions -( - namespace text NOT NULL, - id text NOT NULL, - version text NOT NULL, - draft boolean NOT NULL default true, - data bytea NOT NULL, - created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (namespace, id, version) -); - - -CREATE TABLE api_deployments -( - namespace text NOT NULL, - site text NOT NULL, - host text NOT NULL, - subdomain text, - definition_id text NOT NULL, - definition_version text NOT NULL, - created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (namespace, site, definition_id, definition_version) -); - diff --git a/golem-worker-service-base/tests/db/migration/sqlite/001__init.sql b/golem-worker-service-base/tests/db/migration/sqlite/001__init.sql deleted file mode 100644 index 2aae50576c..0000000000 --- a/golem-worker-service-base/tests/db/migration/sqlite/001__init.sql +++ /dev/null @@ -1,23 +0,0 @@ -CREATE TABLE api_definitions -( - namespace text NOT NULL, - id text NOT NULL, - version text NOT NULL, - draft boolean NOT NULL default true, - data blob NOT NULL, - created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, - PRIMARY KEY (namespace, id, version) -); - - -CREATE TABLE api_deployments -( - namespace text NOT NULL, - site text NOT NULL, - host text NOT NULL, - subdomain text, - definition_id text NOT NULL, - definition_version text NOT NULL, - created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, - PRIMARY KEY (namespace, site, definition_id, definition_version) -); diff --git a/golem-worker-service-base/tests/services_tests.rs b/golem-worker-service-base/tests/services_tests.rs index fc6a023710..77a3969783 100644 --- a/golem-worker-service-base/tests/services_tests.rs +++ b/golem-worker-service-base/tests/services_tests.rs @@ -65,7 +65,7 @@ mod tests { let cli = Cli::default(); let (db_config, _container) = start_docker_postgres(&cli); - db::postgres_migrate(&db_config, "tests/db/migration/postgres") + db::postgres_migrate(&db_config, "../golem-worker-service/db/migration/postgres") .await .unwrap(); @@ -91,7 +91,7 @@ mod tests { max_connections: 10, }; - db::sqlite_migrate(&db_config, "tests/db/migration/sqlite") + db::sqlite_migrate(&db_config, "../golem-worker-service/db/migration/sqlite") .await .unwrap();