From 8896c4bbbac7e96ad039eda78df99a59c9201779 Mon Sep 17 00:00:00 2001 From: Andrew Chen Date: Fri, 27 Sep 2024 01:40:02 +0000 Subject: [PATCH 1/2] fixes --- build.rs | 1 + src/sinks/gcp/bigquery/config.rs | 2 +- src/sinks/gcp/bigquery/mod.rs | 20 +++++++++++--------- src/sinks/gcp/bigquery/request_builder.rs | 6 +++--- src/sinks/gcp/bigquery/service.rs | 18 +++++++++--------- src/sinks/gcp/mod.rs | 1 - 6 files changed, 25 insertions(+), 23 deletions(-) diff --git a/build.rs b/build.rs index 45b8ca5992977..9009d7c314c91 100644 --- a/build.rs +++ b/build.rs @@ -151,6 +151,7 @@ fn main() { "proto/vector/dd_trace.proto", "proto/third-party/google/cloud/bigquery/storage/v1/storage.proto", "proto/third-party/google/pubsub/v1/pubsub.proto", + "proto/third-party/google/rpc/code.proto", "proto/third-party/google/rpc/status.proto", "proto/vector/vector.proto", ], diff --git a/src/sinks/gcp/bigquery/config.rs b/src/sinks/gcp/bigquery/config.rs index 0fda77182994b..d8c03dc6cfc44 100644 --- a/src/sinks/gcp/bigquery/config.rs +++ b/src/sinks/gcp/bigquery/config.rs @@ -5,7 +5,7 @@ use indoc::indoc; use tonic::transport::Channel; use vector_config::configurable_component; -use super::proto::google::cloud::bigquery::storage::v1 as proto; +use super::proto::third_party::google::cloud::bigquery::storage::v1 as proto; use super::request_builder::{BigqueryRequestBuilder, MAX_BATCH_PAYLOAD_SIZE}; use super::service::{AuthInterceptor, BigqueryService}; use super::sink::BigquerySink; diff --git a/src/sinks/gcp/bigquery/mod.rs b/src/sinks/gcp/bigquery/mod.rs index 6936522b00fa8..b92fac01a2864 100644 --- a/src/sinks/gcp/bigquery/mod.rs +++ b/src/sinks/gcp/bigquery/mod.rs @@ -16,18 +16,20 @@ mod sink; #[allow(warnings, clippy::pedantic, clippy::nursery)] pub(crate) mod proto { - pub(crate) mod google { - pub(crate) mod cloud { - pub(crate) mod bigquery { - pub(crate) mod storage { - pub(crate) mod v1 { - tonic::include_proto!("google.cloud.bigquery.storage.v1"); + pub(crate) mod third_party { + pub(crate) mod google { + pub(crate) mod cloud { + pub(crate) mod bigquery { + pub(crate) mod storage { + pub(crate) mod v1 { + tonic::include_proto!("google.cloud.bigquery.storage.v1"); + } } } } - } - pub(crate) mod rpc { - tonic::include_proto!("google.rpc"); + pub(crate) mod rpc { + tonic::include_proto!("google.rpc"); + } } } } diff --git a/src/sinks/gcp/bigquery/request_builder.rs b/src/sinks/gcp/bigquery/request_builder.rs index 6e0b3e985caa8..bfb4208f65729 100644 --- a/src/sinks/gcp/bigquery/request_builder.rs +++ b/src/sinks/gcp/bigquery/request_builder.rs @@ -3,10 +3,10 @@ use codecs::encoding::ProtobufSerializer; use prost::Message; use std::num::NonZeroUsize; use tokio_util::codec::Encoder; -use vector_common::request_metadata::RequestMetadata; -use vector_core::event::Finalizable; +use vector_lib::request_metadata::RequestMetadata; +use vector_lib::event::Finalizable; -use super::proto::google::cloud::bigquery::storage::v1 as proto; +use super::proto::third_party::google::cloud::bigquery::storage::v1 as proto; use super::service::BigqueryRequest; use crate::event::{Event, EventFinalizers}; use crate::sinks::util::metadata::RequestMetadataBuilder; diff --git a/src/sinks/gcp/bigquery/service.rs b/src/sinks/gcp/bigquery/service.rs index 3fec202992ed4..62fccbdb2f51c 100644 --- a/src/sinks/gcp/bigquery/service.rs +++ b/src/sinks/gcp/bigquery/service.rs @@ -7,11 +7,11 @@ use tonic::service::Interceptor; use tonic::transport::Channel; use tonic::{Request, Status}; use tower::Service; -use vector_common::request_metadata::{GroupedCountByteSize, MetaDescriptive, RequestMetadata}; -use vector_core::event::EventStatus; -use vector_core::stream::DriverResponse; +use vector_lib::request_metadata::{GroupedCountByteSize, MetaDescriptive, RequestMetadata}; +use vector_lib::event::EventStatus; +use vector_lib::stream::DriverResponse; -use super::proto::google::cloud::bigquery::storage::v1 as proto; +use super::proto::third_party::google::cloud::bigquery::storage::v1 as proto; use crate::event::{EventFinalizers, Finalizable}; use crate::gcp::GcpAuthenticator; @@ -75,13 +75,13 @@ impl DriverResponse for BigqueryResponse { None => EventStatus::Dropped, Some(proto::append_rows_response::Response::AppendResult(_)) => EventStatus::Delivered, Some(proto::append_rows_response::Response::Error(status)) => { - match super::proto::google::rpc::Code::try_from(status.code) { + match super::proto::third_party::google::rpc::Code::try_from(status.code) { // we really shouldn't be able to get here, but just in case - Ok(super::proto::google::rpc::Code::Ok) => EventStatus::Delivered, + Ok(super::proto::third_party::google::rpc::Code::Ok) => EventStatus::Delivered, // these errors can't be retried because the event payload is almost definitely bad - Ok(super::proto::google::rpc::Code::InvalidArgument) - | Ok(super::proto::google::rpc::Code::NotFound) - | Ok(super::proto::google::rpc::Code::AlreadyExists) => EventStatus::Rejected, + Ok(super::proto::third_party::google::rpc::Code::InvalidArgument) + | Ok(super::proto::third_party::google::rpc::Code::NotFound) + | Ok(super::proto::third_party::google::rpc::Code::AlreadyExists) => EventStatus::Rejected, // everything else can probably be retried _ => EventStatus::Errored, } diff --git a/src/sinks/gcp/mod.rs b/src/sinks/gcp/mod.rs index 26800790b79e2..fd3dbe1ed4501 100644 --- a/src/sinks/gcp/mod.rs +++ b/src/sinks/gcp/mod.rs @@ -4,7 +4,6 @@ use serde::{Deserialize, Serialize}; use vector_lib::configurable::configurable_component; pub mod bigquery; -pub mod chronicle_unstructured; pub mod cloud_storage; pub mod pubsub; pub mod stackdriver; From 44948bd52fdc349fa18a75041bb5d8ddd5297646 Mon Sep 17 00:00:00 2001 From: Andrew Chen Date: Fri, 27 Sep 2024 22:28:43 +0000 Subject: [PATCH 2/2] more fixes --- src/sinks/gcp/bigquery/config.rs | 6 +++--- src/sinks/gcp/bigquery/request_builder.rs | 8 ++++---- src/sinks/gcp/bigquery/sink.rs | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/sinks/gcp/bigquery/config.rs b/src/sinks/gcp/bigquery/config.rs index d8c03dc6cfc44..1b38e82e53a55 100644 --- a/src/sinks/gcp/bigquery/config.rs +++ b/src/sinks/gcp/bigquery/config.rs @@ -1,9 +1,9 @@ -use codecs::encoding::ProtobufSerializerConfig; +use vector_lib::codecs::encoding::ProtobufSerializerConfig; use futures::FutureExt; use http::Uri; use indoc::indoc; use tonic::transport::Channel; -use vector_config::configurable_component; +use vector_lib::configurable::configurable_component; use super::proto::third_party::google::cloud::bigquery::storage::v1 as proto; use super::request_builder::{BigqueryRequestBuilder, MAX_BATCH_PAYLOAD_SIZE}; @@ -75,7 +75,7 @@ pub struct BigqueryConfig { #[serde( default, deserialize_with = "crate::serde::bool_or_struct", - skip_serializing_if = "crate::serde::skip_serializing_if_default" + skip_serializing_if = "crate::serde::is_default" )] acknowledgements: AcknowledgementsConfig, } diff --git a/src/sinks/gcp/bigquery/request_builder.rs b/src/sinks/gcp/bigquery/request_builder.rs index bfb4208f65729..350d82046fede 100644 --- a/src/sinks/gcp/bigquery/request_builder.rs +++ b/src/sinks/gcp/bigquery/request_builder.rs @@ -1,5 +1,5 @@ use bytes::BytesMut; -use codecs::encoding::ProtobufSerializer; +use vector_lib::codecs::encoding::ProtobufSerializer; use prost::Message; use std::num::NonZeroUsize; use tokio_util::codec::Encoder; @@ -22,8 +22,8 @@ pub enum BigqueryRequestBuilderError { ProtobufEncoding { message: String }, // `error` needs to be some concrete type } -impl From for BigqueryRequestBuilderError { - fn from(error: vector_common::Error) -> Self { +impl From for BigqueryRequestBuilderError { + fn from(error: vector_lib::Error) -> Self { BigqueryRequestBuilderError::ProtobufEncoding { message: format!("{:?}", error), } @@ -142,7 +142,7 @@ mod test { use codecs::encoding::{ProtobufSerializerConfig, ProtobufSerializerOptions}; use std::collections::BTreeMap; use std::path::PathBuf; - use vector_core::event::{Event, EventMetadata, LogEvent, Value}; + use vector_lib::event::{Event, EventMetadata, LogEvent, Value}; use super::BigqueryRequestBuilder; use crate::sinks::util::IncrementalRequestBuilder; diff --git a/src/sinks/gcp/bigquery/sink.rs b/src/sinks/gcp/bigquery/sink.rs index a1ffebbce9159..63f2abe79fcfa 100644 --- a/src/sinks/gcp/bigquery/sink.rs +++ b/src/sinks/gcp/bigquery/sink.rs @@ -2,9 +2,9 @@ use futures_util::{ stream::{self, BoxStream}, StreamExt, }; -use vector_core::event::Event; -use vector_core::sink::StreamSink; -use vector_core::stream::BatcherSettings; +use vector_lib::event::Event; +use vector_lib::sink::StreamSink; +use vector_lib::stream::BatcherSettings; use super::request_builder::BigqueryRequestBuilder; use super::service::BigqueryService;