diff --git a/Cargo.lock b/Cargo.lock index 5587e6e3..a95df67a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -791,7 +791,7 @@ dependencies = [ [[package]] name = "dropshot-authorization-header" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/v-api#52699f41d059548ccc27c15ffcf7eb52be033670" +source = "git+https://github.com/oxidecomputer/v-api#a22f845fcc3c75b8caa3adf6f20eaf36207052a1" dependencies = [ "async-trait", "base64 0.22.1", @@ -2249,7 +2249,7 @@ dependencies = [ [[package]] name = "partial-struct" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/partial-struct#7a45b60d01b24a0a140be8de85b1f61eaa6208e3" +source = "git+https://github.com/oxidecomputer/partial-struct#013155bb04b3baebe4ec0b2335cec3674a0b2064" dependencies = [ "proc-macro2", "quote", @@ -4384,7 +4384,7 @@ dependencies = [ [[package]] name = "v-api" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/v-api#52699f41d059548ccc27c15ffcf7eb52be033670" +source = "git+https://github.com/oxidecomputer/v-api#a22f845fcc3c75b8caa3adf6f20eaf36207052a1" dependencies = [ "async-trait", "base64 0.22.1", @@ -4428,7 +4428,7 @@ dependencies = [ [[package]] name = "v-api-installer" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/v-api#52699f41d059548ccc27c15ffcf7eb52be033670" +source = "git+https://github.com/oxidecomputer/v-api#a22f845fcc3c75b8caa3adf6f20eaf36207052a1" dependencies = [ "diesel", "diesel_migrations", @@ -4437,7 +4437,7 @@ dependencies = [ [[package]] name = "v-api-permission-derive" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/v-api#52699f41d059548ccc27c15ffcf7eb52be033670" +source = "git+https://github.com/oxidecomputer/v-api#a22f845fcc3c75b8caa3adf6f20eaf36207052a1" dependencies = [ "heck", "proc-macro2", @@ -4448,7 +4448,7 @@ dependencies = [ [[package]] name = "v-model" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/v-api#52699f41d059548ccc27c15ffcf7eb52be033670" +source = "git+https://github.com/oxidecomputer/v-api#a22f845fcc3c75b8caa3adf6f20eaf36207052a1" dependencies = [ "async-bb8-diesel", "async-trait", @@ -4669,7 +4669,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] diff --git a/rfd-api-spec.json b/rfd-api-spec.json index 1113a919..38cbe202 100644 --- a/rfd-api-spec.json +++ b/rfd-api-spec.json @@ -1941,11 +1941,7 @@ "content": { "application/json": { "schema": { - "title": "Array_of_RfdPdf", - "type": "array", - "items": { - "$ref": "#/components/schemas/RfdPdf" - } + "$ref": "#/components/schemas/RfdRevisionPdf" } } } @@ -2216,11 +2212,7 @@ "content": { "application/json": { "schema": { - "title": "Array_of_RfdPdf", - "type": "array", - "items": { - "$ref": "#/components/schemas/RfdPdf" - } + "$ref": "#/components/schemas/RfdRevisionPdf" } } } @@ -4415,6 +4407,80 @@ "updated_at" ] }, + "RfdRevisionPdf": { + "type": "object", + "properties": { + "authors": { + "nullable": true, + "type": "string" + }, + "commit": { + "$ref": "#/components/schemas/CommitSha" + }, + "committed_at": { + "type": "string", + "format": "date-time" + }, + "content": { + "type": "array", + "items": { + "$ref": "#/components/schemas/RfdPdf" + } + }, + "content_format": { + "$ref": "#/components/schemas/ContentFormat" + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "deleted_at": { + "nullable": true, + "type": "string", + "format": "date-time" + }, + "discussion": { + "nullable": true, + "type": "string" + }, + "id": { + "$ref": "#/components/schemas/TypedUuidForRfdRevisionId" + }, + "labels": { + "nullable": true, + "type": "string" + }, + "rfd_id": { + "$ref": "#/components/schemas/TypedUuidForRfdId" + }, + "sha": { + "$ref": "#/components/schemas/FileSha" + }, + "state": { + "nullable": true, + "type": "string" + }, + "title": { + "type": "string" + }, + "updated_at": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "commit", + "committed_at", + "content", + "content_format", + "created_at", + "id", + "rfd_id", + "sha", + "title", + "updated_at" + ] + }, "RfdState": { "type": "string", "enum": [ diff --git a/rfd-api/src/context.rs b/rfd-api/src/context.rs index 5a5d79be..517eba1e 100644 --- a/rfd-api/src/context.rs +++ b/rfd-api/src/context.rs @@ -17,8 +17,9 @@ use rfd_data::{ use rfd_github::{GitHubError, GitHubNewRfdNumber, GitHubRfdRepo}; use rfd_model::{ schema_ext::{ContentFormat, Visibility}, - storage::{JobStore, RfdFilter, RfdMetaStore, RfdPdfFilter, RfdPdfStore, RfdStorage, RfdStore}, - CommitSha, FileSha, Job, NewJob, Rfd, RfdId, RfdMeta, RfdPdf, RfdRevision, RfdRevisionId, + storage::{JobStore, RfdFilter, RfdMetaStore, RfdPdfsStore, RfdStorage, RfdStore}, + CommitSha, FileSha, Job, NewJob, Rfd, RfdId, RfdMeta, RfdPdf, RfdPdfs, RfdRevision, + RfdRevisionId, }; use rsa::{ pkcs1::{DecodeRsaPrivateKey, EncodeRsaPrivateKey}, @@ -89,8 +90,9 @@ pub enum UpdateRfdContentError { } #[partial(RfdWithoutContent)] +#[partial(RfdWithPdf)] #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] -pub struct RfdWithContent { +pub struct RfdWithRaw { pub id: TypedUuid, pub rfd_number: i32, pub link: Option, @@ -100,6 +102,7 @@ pub struct RfdWithContent { pub authors: Option, pub labels: Option, #[partial(RfdWithoutContent(skip))] + #[partial(RfdWithPdf(retype = Vec))] pub content: String, pub format: ContentFormat, pub sha: FileSha, @@ -108,9 +111,9 @@ pub struct RfdWithContent { pub visibility: Visibility, } -impl From for RfdWithContent { +impl From for RfdWithRaw { fn from(value: Rfd) -> Self { - RfdWithContent { + Self { id: value.id, rfd_number: value.rfd_number, link: value.link, @@ -131,7 +134,7 @@ impl From for RfdWithContent { impl From for RfdWithoutContent { fn from(value: RfdMeta) -> Self { - RfdWithoutContent { + Self { id: value.id, rfd_number: value.rfd_number, link: value.link, @@ -149,6 +152,27 @@ impl From for RfdWithoutContent { } } +impl From for RfdWithPdf { + fn from(value: RfdPdfs) -> Self { + Self { + id: value.id, + rfd_number: value.rfd_number, + link: value.link, + discussion: value.content.discussion, + title: value.content.title, + state: value.content.state, + authors: value.content.authors, + labels: value.content.labels, + content: value.content.content, + format: value.content.content_format, + sha: value.content.sha, + commit: value.content.commit.into(), + committed_at: value.content.committed_at, + visibility: value.visibility, + } + } +} + #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] pub struct PdfEntry { pub source: String, @@ -446,13 +470,46 @@ impl RfdContext { } } + #[instrument(skip(self, caller))] + async fn get_rfd_pdf( + &self, + caller: &Caller, + rfd_number: i32, + revision: Option, + ) -> ResourceResult { + let mut filter = RfdFilter::default().rfd_number(Some(vec![rfd_number])); + filter = match revision { + Some(RfdRevisionIdentifier::Commit(commit)) => filter.commit(Some(vec![commit])), + Some(RfdRevisionIdentifier::Id(revision)) => filter.revision(Some(vec![revision])), + None => filter, + }; + + let rfd = RfdPdfsStore::list(&*self.storage, vec![filter], &ListPagination::unlimited()) + .await + .to_resource_result()? + .pop(); + + if let Some(rfd) = rfd { + if caller.can(&RfdPermission::GetRfdsAll) + || caller.can(&RfdPermission::GetRfd(rfd.rfd_number)) + || rfd.visibility == Visibility::Public + { + Ok(rfd.into()) + } else { + resource_not_found() + } + } else { + resource_not_found() + } + } + #[instrument(skip(self, caller))] pub async fn view_rfd( &self, caller: &Caller, rfd_number: i32, revision: Option, - ) -> ResourceResult { + ) -> ResourceResult { let rfd = self.get_rfd(caller, rfd_number, revision).await?; Ok(rfd.into()) } @@ -493,19 +550,8 @@ impl RfdContext { caller: &Caller, rfd_number: i32, revision: Option, - ) -> ResourceResult, StoreError> { - let rfd = self.get_rfd_meta(caller, rfd_number, revision).await?; - let pdfs = RfdPdfStore::list( - &*self.storage, - vec![RfdPdfFilter::default() - .rfd(Some(vec![rfd.id])) - .rfd_revision(Some(vec![rfd.content.id]))], - &ListPagination::unlimited(), - ) - .await - .to_resource_result()?; - - Ok(pdfs) + ) -> ResourceResult { + self.get_rfd_pdf(caller, rfd_number, revision).await } #[instrument(skip(self, caller, content))] diff --git a/rfd-api/src/endpoints/rfd.rs b/rfd-api/src/endpoints/rfd.rs index 3e4b0996..16968bc3 100644 --- a/rfd-api/src/endpoints/rfd.rs +++ b/rfd-api/src/endpoints/rfd.rs @@ -13,7 +13,7 @@ use rfd_data::{ }; use rfd_model::{ schema_ext::{ContentFormat, Visibility}, - Rfd, RfdPdf, RfdRevisionId, + Rfd, RfdRevisionId, }; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -24,7 +24,7 @@ use v_model::permissions::Caller; use crate::{ caller::CallerExt, - context::{RfdContext, RfdRevisionIdentifier, RfdWithContent, RfdWithoutContent}, + context::{RfdContext, RfdRevisionIdentifier, RfdWithPdf, RfdWithRaw, RfdWithoutContent}, permissions::RfdPermission, search::{MeiliSearchResult, SearchRequest}, util::response::{client_error, internal_error, unauthorized}, @@ -124,7 +124,7 @@ pub async fn view_rfd_meta( pub async fn view_rfd( rqctx: RequestContext, path: Path, -) -> Result, HttpError> { +) -> Result, HttpError> { let ctx = rqctx.context(); let caller = ctx.v_ctx().get_caller(&rqctx).await?; let path = path.into_inner(); @@ -141,7 +141,7 @@ pub async fn view_rfd( pub async fn view_rfd_pdf( rqctx: RequestContext, path: Path, -) -> Result>, HttpError> { +) -> Result, HttpError> { let ctx = rqctx.context(); let caller = ctx.v_ctx().get_caller(&rqctx).await?; let path = path.into_inner(); @@ -208,7 +208,7 @@ pub async fn view_rfd_revision_meta( pub async fn view_rfd_revision( rqctx: RequestContext, path: Path, -) -> Result, HttpError> { +) -> Result, HttpError> { let ctx = rqctx.context(); let caller = ctx.v_ctx().get_caller(&rqctx).await?; let path = path.into_inner(); @@ -225,7 +225,7 @@ pub async fn view_rfd_revision( pub async fn view_rfd_revision_pdf( rqctx: RequestContext, path: Path, -) -> Result>, HttpError> { +) -> Result, HttpError> { let ctx = rqctx.context(); let caller = ctx.v_ctx().get_caller(&rqctx).await?; let path = path.into_inner(); @@ -365,7 +365,7 @@ async fn view_rfd_op( caller: &Caller, number: String, revision: Option, -) -> Result, HttpError> { +) -> Result, HttpError> { if let Ok(rfd_number) = number.parse::() { Ok(HttpResponseOk( ctx.view_rfd(caller, rfd_number, revision).await?, @@ -384,7 +384,7 @@ async fn view_rfd_pdf_op( caller: &Caller, number: String, revision: Option, -) -> Result>, HttpError> { +) -> Result, HttpError> { if let Ok(rfd_number) = number.parse::() { Ok(HttpResponseOk( ctx.view_rfd_pdfs(caller, rfd_number, revision).await?, @@ -427,7 +427,7 @@ async fn view_rfd_discussion_op( caller: &Caller, number: String, revision: Option, -) -> Result, HttpError> { +) -> Result, HttpError> { unimplemented!() } diff --git a/rfd-model/migrations/2025-01-02-190923_add_commited_at_index/down.sql b/rfd-model/migrations/2025-01-02-190923_add_commited_at_index/down.sql new file mode 100644 index 00000000..71770444 --- /dev/null +++ b/rfd-model/migrations/2025-01-02-190923_add_commited_at_index/down.sql @@ -0,0 +1,2 @@ +DROP INDEX rfd_revision_sort_no_content_idx; +DROP INDEX rfd_number_idx; diff --git a/rfd-model/migrations/2025-01-02-190923_add_commited_at_index/up.sql b/rfd-model/migrations/2025-01-02-190923_add_commited_at_index/up.sql new file mode 100644 index 00000000..0f75b0e5 --- /dev/null +++ b/rfd-model/migrations/2025-01-02-190923_add_commited_at_index/up.sql @@ -0,0 +1,2 @@ +CREATE INDEX rfd_revision_sort_no_content_idx ON rfd_revision (rfd_id, committed_at desc) include(id, title, state, discussion, authors, content_format, sha, commit_sha, created_at, updated_at, deleted_at, labels); +CREATE INDEX rfd_number_idx ON rfd (rfd_number); diff --git a/rfd-model/src/db.rs b/rfd-model/src/db.rs index 92aac45c..33006841 100644 --- a/rfd-model/src/db.rs +++ b/rfd-model/src/db.rs @@ -3,14 +3,14 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use chrono::{DateTime, Utc}; -use diesel::{Insertable, Queryable, Selectable}; +use diesel::{prelude::QueryableByName, Insertable, Queryable, Selectable}; use partial_struct::partial; use serde::{Deserialize, Serialize}; use uuid::Uuid; use crate::{ schema::{job, rfd, rfd_pdf, rfd_revision}, - schema_ext::{ContentFormat, PdfSource, Visibility}, + schema_ext::{rfd_pdfs, ContentFormat, PdfSource, Visibility}, }; #[derive(Debug, Deserialize, Serialize, Queryable, Insertable, Selectable)] @@ -25,8 +25,44 @@ pub struct RfdModel { pub visibility: Visibility, } +#[derive(QueryableByName)] +#[diesel(table_name = rfd_pdfs)] +pub struct RfdPdfsRow { + pub id: Uuid, + pub rfd_number: i32, + pub link: Option, + pub created_at: DateTime, + pub updated_at: DateTime, + pub deleted_at: Option>, + pub visibility: Visibility, + pub revision_id: Uuid, + pub revision_rfd_id: Uuid, + pub revision_title: String, + pub revision_state: Option, + pub revision_discussion: Option, + pub revision_authors: Option, + pub pdf_id: Uuid, + pub pdf_rfd_revision_id: Uuid, + pub pdf_source: PdfSource, + pub pdf_link: String, + pub pdf_created_at: DateTime, + pub pdf_updated_at: DateTime, + pub pdf_deleted_at: Option>, + pub pdf_rfd_id: Uuid, + pub pdf_external_id: String, + pub revision_content_format: ContentFormat, + pub revision_sha: String, + pub revision_commit_sha: String, + pub revision_committed_at: DateTime, + pub revision_created_at: DateTime, + pub revision_updated_at: DateTime, + pub revision_deleted_at: Option>, + pub revision_labels: Option, +} + #[partial(RfdRevisionMetaModel)] -#[derive(Debug, Deserialize, Serialize, Queryable, Insertable, Selectable)] +#[partial(RfdRevisionPdfModel)] +#[derive(Debug, Deserialize, Serialize, Queryable, Selectable)] #[diesel(table_name = rfd_revision)] pub struct RfdRevisionModel { pub id: Uuid, @@ -36,6 +72,7 @@ pub struct RfdRevisionModel { pub discussion: Option, pub authors: Option, #[partial(RfdRevisionMetaModel(skip))] + #[partial(RfdRevisionPdfModel(retype = RfdPdfModel))] pub content: String, pub content_format: ContentFormat, pub sha: String, @@ -47,6 +84,49 @@ pub struct RfdRevisionModel { pub labels: Option, } +impl From for (RfdModel, RfdRevisionPdfModel) { + fn from(value: RfdPdfsRow) -> Self { + ( + RfdModel { + id: value.id, + rfd_number: value.rfd_number, + link: value.link, + created_at: value.created_at, + updated_at: value.updated_at, + deleted_at: value.deleted_at, + visibility: value.visibility, + }, + RfdRevisionPdfModel { + id: value.revision_id, + rfd_id: value.revision_rfd_id, + title: value.revision_title, + state: value.revision_state, + discussion: value.revision_discussion, + authors: value.revision_authors, + content: RfdPdfModel { + id: value.pdf_id, + rfd_revision_id: value.pdf_rfd_revision_id, + source: value.pdf_source, + link: value.pdf_link, + created_at: value.pdf_created_at, + updated_at: value.pdf_updated_at, + deleted_at: value.pdf_deleted_at, + rfd_id: value.pdf_rfd_id, + external_id: value.pdf_external_id, + }, + content_format: value.revision_content_format, + sha: value.revision_sha, + commit_sha: value.revision_commit_sha, + committed_at: value.revision_committed_at, + created_at: value.revision_created_at, + updated_at: value.revision_updated_at, + deleted_at: value.revision_deleted_at, + labels: value.revision_labels, + }, + ) + } +} + #[derive(Debug, Deserialize, Serialize, Queryable, Insertable)] #[diesel(table_name = rfd_pdf)] pub struct RfdPdfModel { diff --git a/rfd-model/src/lib.rs b/rfd-model/src/lib.rs index d897d9f5..23b51101 100644 --- a/rfd-model/src/lib.rs +++ b/rfd-model/src/lib.rs @@ -3,7 +3,9 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use chrono::{DateTime, Utc}; -use db::{JobModel, RfdModel, RfdPdfModel, RfdRevisionMetaModel, RfdRevisionModel}; +use db::{ + JobModel, RfdModel, RfdPdfModel, RfdRevisionMetaModel, RfdRevisionModel, RfdRevisionPdfModel, +}; use newtype_uuid::{GenericUuid, TypedUuid, TypedUuidKind, TypedUuidTag}; use partial_struct::partial; use schema_ext::{ContentFormat, PdfSource, Visibility}; @@ -64,6 +66,7 @@ impl TypedUuidKind for RfdId { #[partial(NewRfd)] #[partial(RfdMeta)] +#[partial(RfdPdfs)] #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct Rfd { pub id: TypedUuid, @@ -71,6 +74,7 @@ pub struct Rfd { pub link: Option, #[partial(NewRfd(skip))] #[partial(RfdMeta(retype = RfdRevisionMeta))] + #[partial(RfdPdfs(retype = RfdRevisionPdf))] pub content: RfdRevision, #[partial(NewRfd(skip))] pub created_at: DateTime, @@ -111,6 +115,21 @@ impl From<(RfdModel, RfdRevisionMetaModel)> for RfdMeta { } } +impl From<(RfdModel, RfdRevisionPdfModel)> for RfdPdfs { + fn from((rfd, revision): (RfdModel, RfdRevisionPdfModel)) -> Self { + Self { + id: TypedUuid::from_untyped_uuid(rfd.id), + rfd_number: rfd.rfd_number, + link: rfd.link, + content: revision.into(), + created_at: rfd.created_at, + updated_at: rfd.updated_at, + deleted_at: rfd.deleted_at, + visibility: rfd.visibility, + } + } +} + impl From for NewRfd { fn from(value: RfdMeta) -> Self { Self { @@ -133,6 +152,7 @@ impl TypedUuidKind for RfdRevisionId { #[partial(NewRfdRevision)] #[partial(RfdRevisionMeta)] +#[partial(RfdRevisionPdf)] #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct RfdRevision { pub id: TypedUuid, @@ -143,6 +163,7 @@ pub struct RfdRevision { pub authors: Option, pub labels: Option, #[partial(RfdRevisionMeta(skip))] + #[partial(RfdRevisionPdf(retype = Vec))] pub content: String, pub content_format: ContentFormat, pub sha: FileSha, @@ -199,6 +220,28 @@ impl From for RfdRevisionMeta { } } +impl From for RfdRevisionPdf { + fn from(value: RfdRevisionPdfModel) -> Self { + Self { + id: TypedUuid::from_untyped_uuid(value.id), + rfd_id: TypedUuid::from_untyped_uuid(value.rfd_id), + title: value.title, + state: value.state, + discussion: value.discussion, + authors: value.authors, + labels: value.labels, + content: vec![value.content.into()], + content_format: value.content_format, + sha: value.sha.into(), + commit: value.commit_sha.into(), + committed_at: value.committed_at, + created_at: value.created_at, + updated_at: value.updated_at, + deleted_at: value.deleted_at, + } + } +} + #[derive(JsonSchema)] pub enum RfdPdfId {} impl TypedUuidKind for RfdPdfId { diff --git a/rfd-model/src/schema_ext.rs b/rfd-model/src/schema_ext.rs index f024c200..872ee20c 100644 --- a/rfd-model/src/schema_ext.rs +++ b/rfd-model/src/schema_ext.rs @@ -118,3 +118,41 @@ impl Display for Visibility { } } } + +diesel::table! { + use diesel::sql_types::*; + use crate::schema::sql_types::{RfdPdfSource, RfdContentFormat, RfdVisibility}; + + rfd_pdfs (id) { + id -> Uuid, + rfd_number -> Integer, + link -> Nullable, + created_at -> Timestamptz, + updated_at -> Timestamptz, + deleted_at -> Nullable, + visibility -> RfdVisibility, + revision_id -> Uuid, + revision_rfd_id -> Uuid, + revision_title -> Varchar, + revision_state -> Nullable, + revision_discussion -> Nullable, + revision_authors -> Nullable, + pdf_id -> Uuid, + pdf_rfd_revision_id -> Uuid, + pdf_source -> RfdPdfSource, + pdf_link -> Varchar, + pdf_created_at -> Timestamptz, + pdf_updated_at -> Timestamptz, + pdf_deleted_at -> Nullable, + pdf_rfd_id -> Uuid, + pdf_external_id -> Varchar, + revision_content_format -> RfdContentFormat, + revision_sha -> Varchar, + revision_commit_sha -> Varchar, + revision_committed_at -> Timestamptz, + revision_created_at -> Timestamptz, + revision_updated_at -> Timestamptz, + revision_deleted_at -> Nullable, + revision_labels -> Nullable, + } +} diff --git a/rfd-model/src/storage/mock.rs b/rfd-model/src/storage/mock.rs index cf3f829c..6477ca3c 100644 --- a/rfd-model/src/storage/mock.rs +++ b/rfd-model/src/storage/mock.rs @@ -8,21 +8,24 @@ use std::sync::Arc; use v_model::storage::StoreError; use crate::{ - Job, NewJob, NewRfd, NewRfdPdf, NewRfdRevision, Rfd, RfdId, RfdMeta, RfdPdf, RfdPdfId, + Job, NewJob, NewRfd, NewRfdPdf, NewRfdRevision, Rfd, RfdId, RfdMeta, RfdPdf, RfdPdfId, RfdPdfs, RfdRevision, RfdRevisionId, RfdRevisionMeta, }; use super::{ JobFilter, JobStore, ListPagination, MockJobStore, MockRfdMetaStore, MockRfdPdfStore, - MockRfdRevisionMetaStore, MockRfdRevisionStore, MockRfdStore, RfdFilter, RfdMetaStore, - RfdPdfFilter, RfdPdfStore, RfdRevisionFilter, RfdRevisionMetaStore, RfdRevisionStore, RfdStore, + MockRfdPdfsStore, MockRfdRevisionMetaStore, MockRfdRevisionPdfStore, MockRfdRevisionStore, + MockRfdStore, RfdFilter, RfdMetaStore, RfdPdfFilter, RfdPdfStore, RfdPdfsStore, + RfdRevisionFilter, RfdRevisionMetaStore, RfdRevisionStore, RfdStore, }; pub struct MockStorage { pub rfd_store: Option>, pub rfd_meta_store: Option>, + pub rfd_pdfs_store: Option>, pub rfd_revision_store: Option>, pub rfd_revision_meta_store: Option>, + pub rfd_revision_pdf_store: Option>, pub rfd_pdf_store: Option>, pub job_store: Option>, } @@ -32,8 +35,10 @@ impl MockStorage { Self { rfd_store: None, rfd_meta_store: None, + rfd_pdfs_store: None, rfd_revision_store: None, rfd_revision_meta_store: None, + rfd_revision_pdf_store: None, rfd_pdf_store: None, job_store: None, } @@ -104,6 +109,34 @@ impl RfdMetaStore for MockStorage { } } +#[async_trait] +impl RfdPdfsStore for MockStorage { + async fn get( + &self, + id: TypedUuid, + revision: Option>, + deleted: bool, + ) -> Result, StoreError> { + self.rfd_pdfs_store + .as_ref() + .unwrap() + .get(id, revision, deleted) + .await + } + + async fn list( + &self, + filters: Vec, + pagination: &ListPagination, + ) -> Result, StoreError> { + self.rfd_pdfs_store + .as_ref() + .unwrap() + .list(filters, pagination) + .await + } +} + #[async_trait] impl RfdRevisionStore for MockStorage { async fn get( diff --git a/rfd-model/src/storage/mod.rs b/rfd-model/src/storage/mod.rs index fe63a0fb..3254979e 100644 --- a/rfd-model/src/storage/mod.rs +++ b/rfd-model/src/storage/mod.rs @@ -13,7 +13,8 @@ use v_model::storage::{ListPagination, StoreError}; use crate::{ schema_ext::PdfSource, CommitSha, Job, NewJob, NewRfd, NewRfdPdf, NewRfdRevision, Rfd, RfdId, - RfdMeta, RfdPdf, RfdPdfId, RfdRevision, RfdRevisionId, RfdRevisionMeta, + RfdMeta, RfdPdf, RfdPdfId, RfdPdfs, RfdRevision, RfdRevisionId, RfdRevisionMeta, + RfdRevisionPdf, }; #[cfg(feature = "mock")] @@ -26,6 +27,7 @@ pub trait RfdStorage: + RfdRevisionStore + RfdRevisionMetaStore + RfdPdfStore + + RfdPdfsStore + JobStore + Send + Sync @@ -38,6 +40,7 @@ impl RfdStorage for T where + RfdRevisionStore + RfdRevisionMetaStore + RfdPdfStore + + RfdPdfsStore + JobStore + Send + Sync @@ -121,6 +124,22 @@ pub trait RfdMetaStore { ) -> Result, StoreError>; } +#[cfg_attr(feature = "mock", automock)] +#[async_trait] +pub trait RfdPdfsStore { + async fn get( + &self, + id: TypedUuid, + revision: Option>, + deleted: bool, + ) -> Result, StoreError>; + async fn list( + &self, + filters: Vec, + pagination: &ListPagination, + ) -> Result, StoreError>; +} + // TODO: Make the revision store generic over a revision type. We want to be able to have a metadata // only version of the revision model so that we do not need to always load content from the db @@ -174,11 +193,6 @@ pub trait RfdRevisionStore { filters: Vec, pagination: &ListPagination, ) -> Result, StoreError>; - // async fn list_unique_rfd( - // &self, - // filters: Vec, - // pagination: &ListPagination, - // ) -> Result, StoreError>; async fn upsert(&self, new_revision: NewRfdRevision) -> Result; async fn delete( &self, @@ -199,11 +213,21 @@ pub trait RfdRevisionMetaStore { filters: Vec, pagination: &ListPagination, ) -> Result, StoreError>; - // async fn list_unique_rfd( - // &self, - // filter: RfdRevisionFilter, - // pagination: &ListPagination, - // ) -> Result, StoreError>; +} + +#[cfg_attr(feature = "mock", automock)] +#[async_trait] +pub trait RfdRevisionPdfStore { + async fn get( + &self, + id: &TypedUuid, + deleted: bool, + ) -> Result, StoreError>; + async fn list( + &self, + filters: Vec, + pagination: &ListPagination, + ) -> Result, StoreError>; } #[derive(Debug, Default)] diff --git a/rfd-model/src/storage/postgres.rs b/rfd-model/src/storage/postgres.rs index 37ba2596..eb433133 100644 --- a/rfd-model/src/storage/postgres.rs +++ b/rfd-model/src/storage/postgres.rs @@ -9,27 +9,33 @@ use diesel::{ debug_query, insert_into, pg::Pg, query_dsl::QueryDsl, + sql_query, sql_types::Bool, update, upsert::{excluded, on_constraint}, BoolExpressionMethods, BoxableExpression, ExpressionMethods, SelectableHelper, }; use newtype_uuid::{GenericUuid, TypedUuid}; +use std::collections::{btree_map::Entry, BTreeMap}; use uuid::Uuid; use v_model::storage::postgres::PostgresStore; use crate::{ - db::{JobModel, RfdModel, RfdPdfModel, RfdRevisionMetaModel, RfdRevisionModel}, + db::{ + JobModel, RfdModel, RfdPdfModel, RfdPdfsRow, RfdRevisionMetaModel, RfdRevisionModel, + RfdRevisionPdfModel, + }, schema::{job, rfd, rfd_pdf, rfd_revision}, schema_ext::Visibility, storage::StoreError, - Job, NewJob, NewRfd, NewRfdPdf, NewRfdRevision, Rfd, RfdId, RfdMeta, RfdPdf, RfdPdfId, - RfdRevision, RfdRevisionId, RfdRevisionMeta, + Job, NewJob, NewRfd, NewRfdPdf, NewRfdRevision, Rfd, RfdId, RfdMeta, RfdPdf, RfdPdfId, RfdPdfs, + RfdRevision, RfdRevisionId, RfdRevisionMeta, RfdRevisionPdf, }; use super::{ JobFilter, JobStore, ListPagination, RfdFilter, RfdMetaStore, RfdPdfFilter, RfdPdfStore, - RfdRevisionFilter, RfdRevisionMetaStore, RfdRevisionStore, RfdStore, + RfdPdfsStore, RfdRevisionFilter, RfdRevisionMetaStore, RfdRevisionPdfStore, RfdRevisionStore, + RfdStore, }; #[async_trait] @@ -263,13 +269,15 @@ impl RfdMetaStore for PostgresStore { query = query.filter(predicate); } - let results = query + query = query .offset(pagination.offset) .limit(pagination.limit) .order(( rfd_revision::rfd_id.asc(), rfd_revision::committed_at.desc(), - )) + )); + + let results = query .get_results_async::<(RfdModel, RfdRevisionMetaModel)>(&*self.pool.get().await?) .await?; @@ -279,6 +287,247 @@ impl RfdMetaStore for PostgresStore { } } +#[async_trait] +impl RfdPdfsStore for PostgresStore { + async fn get( + &self, + id: TypedUuid, + revision: Option>, + deleted: bool, + ) -> Result, StoreError> { + let rfd = RfdPdfsStore::list( + self, + vec![RfdFilter::default() + .id(Some(vec![id])) + .revision(revision.map(|rev| vec![rev])) + .deleted(deleted)], + &ListPagination::default().limit(1), + ) + .await?; + Ok(rfd.into_iter().nth(0)) + } + + async fn list( + &self, + filters: Vec, + pagination: &ListPagination, + ) -> Result, StoreError> { + tracing::trace!(?filters, "Lookup RFDs"); + + let mut clauses = vec![]; + let mut bind_count = 1; + + for filter in &filters { + let mut filter_clause = "1=1".to_string(); + + let RfdFilter { + id, + revision, + rfd_number, + commit, + public, + deleted, + } = filter; + + if let Some(ids) = &id { + let id_binds = ids + .iter() + .enumerate() + .map(|(i, _)| format!("${}", bind_count + i)) + .collect::>(); + bind_count = bind_count + id_binds.len(); + filter_clause = filter_clause + &format!(" AND rfd.id IN ({})", id_binds.join(",")); + } + + if let Some(revisions) = &revision { + let revision_binds = revisions + .iter() + .enumerate() + .map(|(i, _)| format!("${}", bind_count + i)) + .collect::>(); + bind_count = bind_count + revision_binds.len(); + filter_clause = filter_clause + + &format!(" AND rfd_revision.id IN ({})", revision_binds.join(",")); + } + + if let Some(rfd_numbers) = &rfd_number { + let rfd_number_binds = rfd_numbers + .iter() + .enumerate() + .map(|(i, _)| format!("${}", bind_count + i)) + .collect::>(); + bind_count = bind_count + rfd_number_binds.len(); + filter_clause = filter_clause + + &format!(" AND rfd.rfd_number.id IN ({})", rfd_number_binds.join(",")); + } + + if let Some(commit_shas) = &commit { + let commit_sha_binds = commit_shas + .iter() + .enumerate() + .map(|(i, _)| format!("${}", bind_count + i)) + .collect::>(); + bind_count = bind_count + commit_sha_binds.len(); + filter_clause = filter_clause + + &format!( + " AND rfd_revision.commit_sha.id IN ({})", + commit_sha_binds.join(",") + ); + } + + if let Some(_) = &public { + bind_count = bind_count + 1; + filter_clause = filter_clause + " AND rfd.public = {}"; + } + + if !deleted { + filter_clause = filter_clause +" AND rfd.deleted_at IS NULL AND rfd_revision.deleted_at IS NULL AND rfd_pdf.deleted_at IS NULL"; + } + + clauses.push(filter_clause); + } + + let where_clause = clauses.join(" OR "); + + let raw_query = format!( + r#"SELECT + rfd.id, + rfd.rfd_number, + rfd.link, + rfd.created_at, + rfd.updated_at, + rfd.deleted_at, + rfd.visibility, + rfd_revision.id AS revision_id, + rfd_revision.rfd_id, + rfd_revision.title, + rfd_revision.state, + rfd_revision.discussion, + rfd_revision.authors, + rfd_pdf.id, + rfd_pdf.rfd_revision_id, + rfd_pdf.source, + rfd_pdf.link, + rfd_pdf.created_at, + rfd_pdf.updated_at, + rfd_pdf.deleted_at, + rfd_pdf.rfd_id, + rfd_pdf.external_id, + rfd_revision.content_format, + rfd_revision.sha, + rfd_revision.commit_sha, + rfd_revision.committed_at, + rfd_revision.created_at, + rfd_revision.updated_at, + rfd_revision.deleted_at, + rfd_revision.labels + FROM + rfd + INNER JOIN + rfd_revision + ON rfd_revision.rfd_id = rfd.id + INNER JOIN + rfd_pdf + ON rfd_revision.id = rfd_pdf.rfd_revision_id + WHERE {} AND + rfd_revision.id = ( + SELECT rfd_revision.id + FROM rfd_revision + WHERE rfd_revision.rfd_id = rfd.id + ORDER BY rfd_revision.committed_at DESC + LIMIT 1 + ) + ORDER BY + rfd_revision.rfd_id ASC, + rfd_revision.committed_at DESC + LIMIT ${} OFFSET ${}"#, + where_clause, + bind_count, + bind_count + 1, + ); + + let mut query = sql_query(raw_query).into_boxed::(); + + for filter in &filters { + let RfdFilter { + id, + revision, + rfd_number, + commit, + public, + .. + } = filter; + + if let Some(ids) = &id { + for id in ids { + tracing::trace!(?id, "Binding id parameter"); + query = query.bind::(id.into_untyped_uuid()); + } + } + + if let Some(revisions) = &revision { + for revision in revisions { + tracing::trace!(?revision, "Binding revision parameter"); + query = query.bind::(revision.into_untyped_uuid()); + } + } + + if let Some(rfd_numbers) = &rfd_number { + for rfd_number in rfd_numbers { + tracing::trace!(?rfd_number, "Binding rfd_number parameter"); + query = query.bind::(*rfd_number); + } + } + + if let Some(commits) = &commit { + for commit in commits { + tracing::trace!(?commit, "Binding commit parameter"); + query = query.bind::(commit.to_string()); + } + } + + if let Some(public) = &public { + tracing::trace!(?public, "Binding public parameter"); + query = query.bind::(*public); + } + } + + query = query + .bind::(pagination.limit as i32) + .bind::(pagination.offset as i32); + + let rows = query + .get_results_async::(&*self.pool.get().await?) + .await?; + let results = rows + .into_iter() + .map(|row| <(RfdModel, RfdRevisionPdfModel)>::from(row)) + .fold( + BTreeMap::, RfdPdfs>::default(), + |mut map, (rfd_model, revision_model)| { + let entry = map.entry(TypedUuid::from_untyped_uuid(rfd_model.id)); + match entry { + Entry::Occupied(mut existing) => { + existing + .get_mut() + .content + .content + .push(revision_model.content.into()); + } + Entry::Vacant(empty) => { + empty.insert((rfd_model, revision_model).into()); + } + }; + map + }, + ); + + tracing::trace!(count = ?results.len(), "Found RFDs"); + + Ok(results.into_values().collect()) + } +} + #[async_trait] impl RfdRevisionStore for PostgresStore { async fn get( @@ -523,6 +772,136 @@ impl RfdRevisionMetaStore for PostgresStore { } } +#[async_trait] +impl RfdRevisionPdfStore for PostgresStore { + async fn get( + &self, + id: &TypedUuid, + deleted: bool, + ) -> Result, StoreError> { + let user = RfdRevisionPdfStore::list( + self, + vec![RfdRevisionFilter::default() + .id(Some(vec![*id])) + .deleted(deleted)], + &ListPagination::default().limit(1), + ) + .await?; + Ok(user.into_iter().nth(0)) + } + + async fn list( + &self, + filters: Vec, + pagination: &ListPagination, + ) -> Result, StoreError> { + let mut query = rfd_revision::table + .inner_join(rfd_pdf::table) + .select(( + rfd_revision::id, + rfd_revision::rfd_id, + rfd_revision::title, + rfd_revision::state, + rfd_revision::discussion, + rfd_revision::authors, + ( + rfd_pdf::id, + rfd_pdf::rfd_revision_id, + rfd_pdf::source, + rfd_pdf::link, + rfd_pdf::created_at, + rfd_pdf::updated_at, + rfd_pdf::deleted_at, + rfd_pdf::rfd_id, + rfd_pdf::external_id, + ), + rfd_revision::content_format, + rfd_revision::sha, + rfd_revision::commit_sha, + rfd_revision::committed_at, + rfd_revision::created_at, + rfd_revision::updated_at, + rfd_revision::deleted_at, + rfd_revision::labels, + )) + .into_boxed(); + + tracing::trace!(?filters, "Lookup RFD revision pdf"); + + let filter_predicates = filters + .into_iter() + .map(|filter| { + let mut predicates: Vec>> = vec![]; + let RfdRevisionFilter { + id, + rfd, + commit, + deleted, + } = filter; + + if let Some(id) = id { + predicates.push(Box::new( + rfd_revision::id.eq_any(id.into_iter().map(GenericUuid::into_untyped_uuid)), + )); + } + + if let Some(rfd) = rfd { + predicates.push(Box::new( + rfd_revision::rfd_id + .eq_any(rfd.into_iter().map(GenericUuid::into_untyped_uuid)), + )); + } + + if let Some(commit) = commit { + predicates.push(Box::new( + rfd_revision::commit_sha.eq_any(commit.into_iter().map(|sha| sha.0)), + )); + } + + if !deleted { + predicates.push(Box::new(rfd_revision::deleted_at.is_null())); + } + + predicates + }) + .collect::>(); + + if let Some(predicate) = flatten_predicates(filter_predicates) { + query = query.filter(predicate); + } + + let query = query + .offset(pagination.offset) + .limit(pagination.limit) + .order(rfd_revision::committed_at.desc()); + + tracing::info!(query = ?debug_query(&query), "Run list rfd pdf"); + + let results = query + .get_results_async::(&*self.pool.get().await?) + .await?; + let revisions = results.into_iter().fold( + BTreeMap::, RfdRevisionPdf>::default(), + |mut map, revision| { + let entry = map.entry(TypedUuid::from_untyped_uuid(revision.id)); + match entry { + Entry::Occupied(mut existing) => { + existing.get_mut().content.push(revision.content.into()); + } + Entry::Vacant(empty) => { + empty.insert(revision.into()); + } + }; + map + }, + ); + + tracing::trace!(count = ?revisions.len(), "Found RFD revision metadata"); + + Ok(revisions.into_values().collect::>()) + } +} + #[async_trait] impl RfdPdfStore for PostgresStore { async fn get(