From 46a89dcd773ad464a97511047d3e3b23e97aa99e Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Wed, 21 Aug 2024 20:20:41 +0200 Subject: [PATCH 1/4] component with created_at --- .../proto/golem/component/component.proto | 2 + golem-cli/src/model/component.rs | 3 ++ golem-cli/src/model/invoke_result_view.rs | 2 + golem-component-service-base/Cargo.toml | 2 + golem-component-service-base/src/model.rs | 6 +++ .../src/repo/component.rs | 43 +++++++++++++------ .../src/service/component.rs | 7 ++- golem-service-base/Cargo.toml | 1 + golem-service-base/src/model.rs | 11 +++++ golem-worker-service-base/Cargo.toml | 1 + .../src/service/component/default.rs | 2 + .../tests/services_tests.rs | 2 + openapi/golem-service.yaml | 4 ++ 13 files changed, 71 insertions(+), 15 deletions(-) diff --git a/golem-api-grpc/proto/golem/component/component.proto b/golem-api-grpc/proto/golem/component/component.proto index 9dd5c06088..ab899b684c 100644 --- a/golem-api-grpc/proto/golem/component/component.proto +++ b/golem-api-grpc/proto/golem/component/component.proto @@ -5,6 +5,7 @@ package golem.component; import "golem/common/project_id.proto"; import "golem/component/component_metadata.proto"; import "golem/component/versioned_component_id.proto"; +import "google/protobuf/timestamp.proto"; message Component { VersionedComponentId versioned_component_id = 1; @@ -12,4 +13,5 @@ message Component { uint64 component_size = 5; ComponentMetadata metadata = 6; golem.common.ProjectId project_id = 7; + google.protobuf.Timestamp created_at = 8; } diff --git a/golem-cli/src/model/component.rs b/golem-cli/src/model/component.rs index 4a2ed9628f..864f6d73d7 100644 --- a/golem-cli/src/model/component.rs +++ b/golem-cli/src/model/component.rs @@ -21,6 +21,7 @@ pub struct Component { pub component_size: u64, pub metadata: ComponentMetadata, pub project_id: Option, + pub created_at: chrono::DateTime, } impl From for Component { @@ -30,6 +31,7 @@ impl From for Component { component_name, component_size, metadata, + created_at, } = value; Component { @@ -38,6 +40,7 @@ impl From for Component { component_size, metadata, project_id: None, + created_at, } } } diff --git a/golem-cli/src/model/invoke_result_view.rs b/golem-cli/src/model/invoke_result_view.rs index 6828696fb7..56811ebc0a 100644 --- a/golem-cli/src/model/invoke_result_view.rs +++ b/golem-cli/src/model/invoke_result_view.rs @@ -97,6 +97,7 @@ impl InvokeResultView { #[cfg(test)] mod tests { + use chrono::Utc; use golem_wasm_rpc::protobuf::type_annotated_value::TypeAnnotatedValue; use golem_wasm_rpc::protobuf::TypeAnnotatedValue as RootTypeAnnotatedValue; use golem_wasm_rpc::protobuf::TypedTuple; @@ -156,6 +157,7 @@ mod tests { memories: vec![], }, project_id: None, + created_at: Utc::now(), }; InvokeResultView::try_parse_or_json( diff --git a/golem-component-service-base/Cargo.toml b/golem-component-service-base/Cargo.toml index 46184f7e01..2912be0621 100644 --- a/golem-component-service-base/Cargo.toml +++ b/golem-component-service-base/Cargo.toml @@ -15,8 +15,10 @@ anyhow = { workspace = true } async-trait = { workspace = true } bincode = { workspace = true } bytes = { workspace = true } +chrono = { workspace = true } http_02 = { workspace = true } prost = { workspace = true } +prost-types = { workspace = true } serde = { workspace = true } sqlx = { workspace = true, features = [ "runtime-tokio", diff --git a/golem-component-service-base/src/model.rs b/golem-component-service-base/src/model.rs index 6a1f80aa3f..25429c8839 100644 --- a/golem-component-service-base/src/model.rs +++ b/golem-component-service-base/src/model.rs @@ -1,6 +1,7 @@ use golem_common::model::component_metadata::ComponentMetadata; use golem_service_base::model::{ComponentName, VersionedComponentId}; use serde::{Deserialize, Serialize}; +use std::time::SystemTime; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct Component { @@ -9,6 +10,7 @@ pub struct Component { pub component_name: ComponentName, pub component_size: u64, pub metadata: ComponentMetadata, + pub created_at: chrono::DateTime, } impl Component { @@ -31,6 +33,7 @@ impl From> for golem_service_base::model::Compon component_name: value.component_name, component_size: value.component_size, metadata: value.metadata, + created_at: value.created_at, } } } @@ -43,6 +46,9 @@ impl From> for golem_api_grpc::proto::golem::com component_size: value.component_size, metadata: Some(value.metadata.into()), project_id: None, + created_at: Some(prost_types::Timestamp::from(SystemTime::from( + value.created_at, + ))), } } } diff --git a/golem-component-service-base/src/repo/component.rs b/golem-component-service-base/src/repo/component.rs index 46a85eccea..1a4b331fca 100644 --- a/golem-component-service-base/src/repo/component.rs +++ b/golem-component-service-base/src/repo/component.rs @@ -35,6 +35,7 @@ pub struct ComponentRecord { pub size: i32, pub version: i64, pub metadata: Vec, + pub created_at: chrono::DateTime, } impl TryFrom for Component @@ -56,6 +57,7 @@ where component_size: value.size as u64, metadata, versioned_component_id, + created_at: value.created_at, }) } } @@ -84,6 +86,7 @@ where size: value.component_size as i32, version: value.versioned_component_id.version as i64, metadata: metadata.into(), + created_at: value.created_at, }) } } @@ -167,15 +170,16 @@ impl ComponentRepo for DbComponentRepo { sqlx::query( r#" INSERT INTO component_versions - (component_id, version, size, metadata) + (component_id, version, size, metadata, created_at) VALUES - ($1, $2, $3, $4) + ($1, $2, $3, $4, $5) "#, ) .bind(component.component_id) .bind(component.version) .bind(component.size) .bind(component.metadata.clone()) + .bind(component.created_at) .execute(&mut *transaction) .await?; @@ -193,7 +197,8 @@ impl ComponentRepo for DbComponentRepo { c.component_id AS component_id, cv.version AS version, cv.size AS size, - cv.metadata AS metadata + cv.metadata AS metadata, + cv.created_at AS created_at FROM components c JOIN component_versions cv ON c.component_id = cv.component_id WHERE c.component_id = $1 @@ -214,7 +219,8 @@ impl ComponentRepo for DbComponentRepo { c.component_id AS component_id, cv.version AS version, cv.size AS size, - cv.metadata AS metadata + cv.metadata AS metadata, + cv.created_at AS created_at FROM components c JOIN component_versions cv ON c.component_id = cv.component_id WHERE c.namespace = $1 @@ -238,7 +244,8 @@ impl ComponentRepo for DbComponentRepo { c.component_id AS component_id, cv.version AS version, cv.size AS size, - cv.metadata AS metadata + cv.metadata AS metadata, + cv.created_at AS created_at FROM components c JOIN component_versions cv ON c.component_id = cv.component_id WHERE c.component_id = $1 @@ -264,7 +271,8 @@ impl ComponentRepo for DbComponentRepo { c.component_id AS component_id, cv.version AS version, cv.size AS size, - cv.metadata AS metadata + cv.metadata AS metadata, + cv.created_at AS created_at FROM components c JOIN component_versions cv ON c.component_id = cv.component_id WHERE c.component_id = $1 AND cv.version = $2 @@ -290,7 +298,8 @@ impl ComponentRepo for DbComponentRepo { c.component_id AS component_id, cv.version AS version, cv.size AS size, - cv.metadata AS metadata + cv.metadata AS metadata, + cv.created_at AS created_at FROM components c JOIN component_versions cv ON c.component_id = cv.component_id WHERE c.namespace = $1 AND c.name = $2 @@ -384,15 +393,16 @@ impl ComponentRepo for DbComponentRepo { sqlx::query( r#" INSERT INTO component_versions - (component_id, version, size, metadata) + (component_id, version, size, metadata, created_at) VALUES - ($1, $2, $3, $4) + ($1, $2, $3, $4, $5) "#, ) .bind(component.component_id) .bind(component.version) .bind(component.size) .bind(component.metadata.clone()) + .bind(component.created_at) .execute(&mut *transaction) .await?; @@ -410,7 +420,8 @@ impl ComponentRepo for DbComponentRepo { c.component_id AS component_id, cv.version AS version, cv.size AS size, - cv.metadata AS metadata + cv.metadata AS metadata, + cv.created_at::timestamptz AS created_at FROM components c JOIN component_versions cv ON c.component_id = cv.component_id WHERE c.component_id = $1 @@ -431,7 +442,8 @@ impl ComponentRepo for DbComponentRepo { c.component_id AS component_id, cv.version AS version, cv.size AS size, - cv.metadata AS metadata + cv.metadata AS metadata, + cv.created_at::timestamptz AS created_at FROM components c JOIN component_versions cv ON c.component_id = cv.component_id WHERE c.namespace = $1 @@ -455,7 +467,8 @@ impl ComponentRepo for DbComponentRepo { c.component_id AS component_id, cv.version AS version, cv.size AS size, - cv.metadata AS metadata + cv.metadata AS metadata, + cv.created_at::timestamptz AS created_at FROM components c JOIN component_versions cv ON c.component_id = cv.component_id WHERE c.component_id = $1 @@ -481,7 +494,8 @@ impl ComponentRepo for DbComponentRepo { c.component_id AS component_id, cv.version AS version, cv.size AS size, - cv.metadata AS metadata + cv.metadata AS metadata, + cv.created_at::timestamptz AS created_at FROM components c JOIN component_versions cv ON c.component_id = cv.component_id WHERE c.component_id = $1 AND cv.version = $2 @@ -507,7 +521,8 @@ impl ComponentRepo for DbComponentRepo { c.component_id AS component_id, cv.version AS version, cv.size AS size, - cv.metadata AS metadata + cv.metadata AS metadata, + cv.created_at::timestamptz AS created_at FROM components c JOIN component_versions cv ON c.component_id = cv.component_id WHERE c.namespace = $1 AND c.name = $2 diff --git a/golem-component-service-base/src/service/component.rs b/golem-component-service-base/src/service/component.rs index 316185b890..673d2b3361 100644 --- a/golem-component-service-base/src/service/component.rs +++ b/golem-component-service-base/src/service/component.rs @@ -18,6 +18,7 @@ use std::sync::Arc; use crate::service::component_compilation::ComponentCompilationService; use crate::service::component_processor::process_component; use async_trait::async_trait; +use chrono::Utc; use golem_common::model::component_metadata::ComponentProcessingError; use golem_common::model::ComponentId; use tap::TapFallible; @@ -82,6 +83,7 @@ where component_name: component_name.clone(), component_size: data.len() as u64, metadata, + created_at: Utc::now(), versioned_component_id, }) } @@ -235,7 +237,7 @@ where namespace: &Namespace, ) -> Result, ComponentError> { info!(namespace = %namespace, "Update component"); - + let created_at = Utc::now(); let metadata = process_component(&data).map_err(ComponentError::ComponentProcessingError)?; @@ -266,6 +268,7 @@ where let component = Component { component_size, metadata, + created_at, ..next_component }; let record = component @@ -607,6 +610,7 @@ impl ComponentService component_id: component_id.clone(), version: 0, }, + created_at: Utc::now(), }; Ok(fake_component) @@ -631,6 +635,7 @@ impl ComponentService component_id: component_id.clone(), version: 0, }, + created_at: Utc::now(), }; Ok(fake_component) diff --git a/golem-service-base/Cargo.toml b/golem-service-base/Cargo.toml index ad075a28d1..f90c2796cb 100644 --- a/golem-service-base/Cargo.toml +++ b/golem-service-base/Cargo.toml @@ -29,6 +29,7 @@ humantime-serde = { workspace = true } hyper = { workspace = true } num-traits = { workspace = true } poem-openapi = { workspace = true } +prost-types = { workspace = true } rand = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/golem-service-base/src/model.rs b/golem-service-base/src/model.rs index 96f1110148..9a65880803 100644 --- a/golem-service-base/src/model.rs +++ b/golem-service-base/src/model.rs @@ -20,6 +20,7 @@ use golem_common::model::{ use golem_wasm_rpc::protobuf::type_annotated_value::TypeAnnotatedValue; use poem_openapi::{Enum, NewType, Object, Union}; use serde::{Deserialize, Serialize}; +use std::time::SystemTime; use std::{collections::HashMap, fmt::Display, fmt::Formatter}; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Object)] @@ -1500,6 +1501,7 @@ pub struct Component { pub component_name: ComponentName, pub component_size: u64, pub metadata: ComponentMetadata, + pub created_at: chrono::DateTime, } impl TryFrom for Component { @@ -1508,6 +1510,11 @@ impl TryFrom for Component { fn try_from( value: golem_api_grpc::proto::golem::component::Component, ) -> Result { + let created_at = value + .created_at + .ok_or("Missing created_at") + .and_then(|t| SystemTime::try_from(t).map_err(|_| "Failed to convert timestamp"))?; + Ok(Self { versioned_component_id: value .versioned_component_id @@ -1516,6 +1523,7 @@ impl TryFrom for Component { component_name: ComponentName(value.component_name), component_size: value.component_size, metadata: value.metadata.ok_or("Missing metadata")?.try_into()?, + created_at: created_at.into(), }) } } @@ -1528,6 +1536,9 @@ impl From for golem_api_grpc::proto::golem::component::Component { component_size: value.component_size, metadata: Some(value.metadata.into()), project_id: None, + created_at: Some(prost_types::Timestamp::from(SystemTime::from( + value.created_at, + ))), } } } diff --git a/golem-worker-service-base/Cargo.toml b/golem-worker-service-base/Cargo.toml index 19d7fea185..96f8993aec 100644 --- a/golem-worker-service-base/Cargo.toml +++ b/golem-worker-service-base/Cargo.toml @@ -21,6 +21,7 @@ anyhow = { workspace = true } async-trait = { workspace = true } bincode = { workspace = true } bytes = { workspace = true } +chrono = { workspace = true } derive_more = { workspace = true } figment = { workspace = true } futures = { workspace = true } diff --git a/golem-worker-service-base/src/service/component/default.rs b/golem-worker-service-base/src/service/component/default.rs index 719ab9a0ba..5698419e75 100644 --- a/golem-worker-service-base/src/service/component/default.rs +++ b/golem-worker-service-base/src/service/component/default.rs @@ -1,4 +1,5 @@ use async_trait::async_trait; +use chrono::Utc; use http::Uri; use tonic::transport::Channel; @@ -215,6 +216,7 @@ impl ComponentServiceNoop { producers: vec![], memories: vec![], }, + created_at: Utc::now(), } } } diff --git a/golem-worker-service-base/tests/services_tests.rs b/golem-worker-service-base/tests/services_tests.rs index ea588665b5..317cc3a85f 100644 --- a/golem-worker-service-base/tests/services_tests.rs +++ b/golem-worker-service-base/tests/services_tests.rs @@ -28,6 +28,7 @@ mod tests { }; use std::sync::Arc; + use chrono::Utc; use testcontainers::clients::Cli; use testcontainers::{Container, RunnableImage}; use testcontainers_modules::postgres::Postgres; @@ -139,6 +140,7 @@ mod tests { producers: vec![], memories: vec![], }, + created_at: Utc::now() } } diff --git a/openapi/golem-service.yaml b/openapi/golem-service.yaml index 245d618069..4c565fec29 100644 --- a/openapi/golem-service.yaml +++ b/openapi/golem-service.yaml @@ -3758,11 +3758,15 @@ components: format: uint64 metadata: $ref: '#/components/schemas/ComponentMetadata' + createdAt: + type: string + format: date-time required: - versionedComponentId - componentName - componentSize - metadata + - createdAt ComponentMetadata: type: object properties: From f6506b0260616e9f601e742bbb9092bfe094f12c Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Wed, 21 Aug 2024 21:25:48 +0200 Subject: [PATCH 2/4] rebase --- Cargo.lock | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index dd2ab83885..809c37bda1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3546,6 +3546,7 @@ dependencies = [ "async-trait", "bincode", "bytes 1.7.1", + "chrono", "criterion", "fastrand 2.1.0", "golem-api-grpc", @@ -3554,6 +3555,7 @@ dependencies = [ "golem-wasm-ast", "http 0.2.12", "prost", + "prost-types", "serde 1.0.207", "sqlx", "tap", @@ -3649,6 +3651,7 @@ dependencies = [ "num-traits 0.2.19", "poem-openapi", "proptest", + "prost-types", "rand", "serde 1.0.207", "serde_json", @@ -3997,6 +4000,7 @@ dependencies = [ "async-trait", "bincode", "bytes 1.7.1", + "chrono", "criterion", "derive_more", "fastrand 2.1.0", From 98f3b1139ff619976971c3c08d8494ab2489660b Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Wed, 21 Aug 2024 21:39:01 +0200 Subject: [PATCH 3/4] fmt fix --- golem-worker-service-base/tests/services_tests.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/golem-worker-service-base/tests/services_tests.rs b/golem-worker-service-base/tests/services_tests.rs index 317cc3a85f..7089ba03b6 100644 --- a/golem-worker-service-base/tests/services_tests.rs +++ b/golem-worker-service-base/tests/services_tests.rs @@ -27,8 +27,8 @@ mod tests { HttpApiDefinitionValidator, RouteValidationError, }; - use std::sync::Arc; use chrono::Utc; + use std::sync::Arc; use testcontainers::clients::Cli; use testcontainers::{Container, RunnableImage}; use testcontainers_modules::postgres::Postgres; @@ -140,7 +140,7 @@ mod tests { producers: vec![], memories: vec![], }, - created_at: Utc::now() + created_at: Utc::now(), } } From f26c12849b1ac28c62ff478e2060372615457908 Mon Sep 17 00:00:00 2001 From: Peter Kotula Date: Thu, 22 Aug 2024 10:46:48 +0200 Subject: [PATCH 4/4] component created_at optional in API response --- golem-cli/src/model/component.rs | 2 +- golem-cli/src/model/invoke_result_view.rs | 2 +- golem-component-service-base/src/model.rs | 2 +- golem-service-base/src/model.rs | 26 ++++++++++--------- .../src/service/component/default.rs | 2 +- .../tests/services_tests.rs | 2 +- openapi/golem-service.yaml | 1 - 7 files changed, 19 insertions(+), 18 deletions(-) diff --git a/golem-cli/src/model/component.rs b/golem-cli/src/model/component.rs index 864f6d73d7..40ad77d216 100644 --- a/golem-cli/src/model/component.rs +++ b/golem-cli/src/model/component.rs @@ -21,7 +21,7 @@ pub struct Component { pub component_size: u64, pub metadata: ComponentMetadata, pub project_id: Option, - pub created_at: chrono::DateTime, + pub created_at: Option>, } impl From for Component { diff --git a/golem-cli/src/model/invoke_result_view.rs b/golem-cli/src/model/invoke_result_view.rs index 56811ebc0a..aee06053b4 100644 --- a/golem-cli/src/model/invoke_result_view.rs +++ b/golem-cli/src/model/invoke_result_view.rs @@ -157,7 +157,7 @@ mod tests { memories: vec![], }, project_id: None, - created_at: Utc::now(), + created_at: Some(Utc::now()), }; InvokeResultView::try_parse_or_json( diff --git a/golem-component-service-base/src/model.rs b/golem-component-service-base/src/model.rs index 25429c8839..347db6549f 100644 --- a/golem-component-service-base/src/model.rs +++ b/golem-component-service-base/src/model.rs @@ -33,7 +33,7 @@ impl From> for golem_service_base::model::Compon component_name: value.component_name, component_size: value.component_size, metadata: value.metadata, - created_at: value.created_at, + created_at: Some(value.created_at), } } } diff --git a/golem-service-base/src/model.rs b/golem-service-base/src/model.rs index 9a65880803..298b5a2e14 100644 --- a/golem-service-base/src/model.rs +++ b/golem-service-base/src/model.rs @@ -1430,13 +1430,13 @@ impl From for golem_api_grpc::proto::golem::worker::v1::worker_execu golem_api_grpc::proto::golem::worker::v1::worker_execution_error::Error::UnexpectedOplogEntry(err.into()) } GolemError::RuntimeError(err) => { - golem_api_grpc::proto::golem::worker::v1::worker_execution_error::Error::RuntimeError(err.into()) + golem_api_grpc::proto::golem::worker::v1::worker_execution_error::Error::RuntimeError(err.into()) } GolemError::InvalidShardId(err) => { golem_api_grpc::proto::golem::worker::v1::worker_execution_error::Error::InvalidShardId(err.into()) } GolemError::PreviousInvocationFailed(err) => { - golem_api_grpc::proto::golem::worker::v1::worker_execution_error::Error::PreviousInvocationFailed(err.into()) + golem_api_grpc::proto::golem::worker::v1::worker_execution_error::Error::PreviousInvocationFailed(err.into()) } GolemError::PreviousInvocationExited(err) => { golem_api_grpc::proto::golem::worker::v1::worker_execution_error::Error::PreviousInvocationExited(err.into()) @@ -1501,7 +1501,7 @@ pub struct Component { pub component_name: ComponentName, pub component_size: u64, pub metadata: ComponentMetadata, - pub created_at: chrono::DateTime, + pub created_at: Option>, } impl TryFrom for Component { @@ -1510,11 +1510,13 @@ impl TryFrom for Component { fn try_from( value: golem_api_grpc::proto::golem::component::Component, ) -> Result { - let created_at = value - .created_at - .ok_or("Missing created_at") - .and_then(|t| SystemTime::try_from(t).map_err(|_| "Failed to convert timestamp"))?; - + let created_at = match value.created_at { + Some(t) => { + let t = SystemTime::try_from(t).map_err(|_| "Failed to convert timestamp")?; + Some(t.into()) + } + None => None, + }; Ok(Self { versioned_component_id: value .versioned_component_id @@ -1523,7 +1525,7 @@ impl TryFrom for Component { component_name: ComponentName(value.component_name), component_size: value.component_size, metadata: value.metadata.ok_or("Missing metadata")?.try_into()?, - created_at: created_at.into(), + created_at, }) } } @@ -1536,9 +1538,9 @@ impl From for golem_api_grpc::proto::golem::component::Component { component_size: value.component_size, metadata: Some(value.metadata.into()), project_id: None, - created_at: Some(prost_types::Timestamp::from(SystemTime::from( - value.created_at, - ))), + created_at: value + .created_at + .map(|t| prost_types::Timestamp::from(SystemTime::from(t))), } } } diff --git a/golem-worker-service-base/src/service/component/default.rs b/golem-worker-service-base/src/service/component/default.rs index 5698419e75..26773f9977 100644 --- a/golem-worker-service-base/src/service/component/default.rs +++ b/golem-worker-service-base/src/service/component/default.rs @@ -216,7 +216,7 @@ impl ComponentServiceNoop { producers: vec![], memories: vec![], }, - created_at: Utc::now(), + created_at: Some(Utc::now()), } } } diff --git a/golem-worker-service-base/tests/services_tests.rs b/golem-worker-service-base/tests/services_tests.rs index 7089ba03b6..13da26eaec 100644 --- a/golem-worker-service-base/tests/services_tests.rs +++ b/golem-worker-service-base/tests/services_tests.rs @@ -140,7 +140,7 @@ mod tests { producers: vec![], memories: vec![], }, - created_at: Utc::now(), + created_at: Some(Utc::now()), } } diff --git a/openapi/golem-service.yaml b/openapi/golem-service.yaml index 4c565fec29..35eb08663d 100644 --- a/openapi/golem-service.yaml +++ b/openapi/golem-service.yaml @@ -3766,7 +3766,6 @@ components: - componentName - componentSize - metadata - - createdAt ComponentMetadata: type: object properties: