From 52bfe7d0824fe023cba47cb6ba70f48f3b8d7154 Mon Sep 17 00:00:00 2001 From: Cedric Kienzler Date: Thu, 28 Nov 2024 17:27:50 +0100 Subject: [PATCH 1/6] feat(github/star): Add support for backing up starred GitHub repos This commit adds support for backing up starred repositories, using a backup rule with `kind: github/star`. To achieve this, the src/sources/github_repo.rs file was modified to support both the https://api.github.com/users/{USER}/repos and the https://api.github.com/users/{USER}/repos API-endpoint. This is made possible by introducing separate constructors for `repo` and `star` which set a new `kind` enum-member. `kind` is used to return the kind string in `fn kind()` as well as to construct the correct API-endpoint in `fn load`. Documentation and examples for `kind: github/star` were added, as well as appropriate unit-testing. To cut down on "magic string" usage, the enum used for `kind` is specified in src/helpers/github.rs and also includes the release kind, making it possible to use the enum throughout the code, instead of having to rely on the raw strings. Additionally, this commit adds on to the documentation in README.md by providing better documentation of the available tokens in the `filter` expressions. --- examples/config.yaml | 2 + src/helpers/github.rs | 63 +++++++++++++++++++ src/main.rs | 31 ++++++++- src/sources/github_releases.rs | 19 +++--- src/sources/github_repo.rs | 112 ++++++++++++++++++++++++++++----- 5 files changed, 201 insertions(+), 26 deletions(-) diff --git a/examples/config.yaml b/examples/config.yaml index aeca02b..f0cf3a6 100644 --- a/examples/config.yaml +++ b/examples/config.yaml @@ -10,3 +10,5 @@ backups: - kind: github/release from: orgs/SierraSoftworks filter: repo.public && !release.prerelease && !artifact.source-code + - kind: github/star + from: users/notheotherben diff --git a/src/helpers/github.rs b/src/helpers/github.rs index 1dd9d3f..baa7702 100644 --- a/src/helpers/github.rs +++ b/src/helpers/github.rs @@ -568,6 +568,57 @@ impl MetadataSource for GitHubReleaseAsset { } } +#[allow(dead_code)] +#[derive(PartialEq)] +#[derive(Debug)] +pub enum GitHubKind { + Repo, + Star, + Release, +} + +impl GitHubKind { + pub fn as_str(&self) -> &'static str { + match self { + GitHubKind::Repo => "github/repo", + GitHubKind::Star => "github/star", + GitHubKind::Release => "github/release", + } + } + + pub fn api_endpoint(&self) -> &'static str { + match self { + GitHubKind::Repo => "repos", + GitHubKind::Star => "starred", + GitHubKind::Release => "repos", + } + } +} + +impl serde::Serialize for GitHubKind { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> serde::Deserialize<'de> for GitHubKind { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s: &str = serde::Deserialize::deserialize(deserializer)?; + match s { + "github/repo" => Ok(GitHubKind::Repo), + "github/star" => Ok(GitHubKind::Star), + "github/release" => Ok(GitHubKind::Release), + _ => Err(serde::de::Error::custom(format!("Invalid kind: {}", s))), + } + } +} + #[cfg(test)] mod tests { use std::path::PathBuf; @@ -686,4 +737,16 @@ mod tests { }) .unwrap_or(Credentials::None) } + + #[rstest] + #[case("github/repo", GitHubKind::Repo, "repos",)] + #[case("github/star", GitHubKind::Star, "starred",)] + #[case("github/release", GitHubKind::Release, "repos",)] + fn test_deserialize_gh_repo_kind(#[case] kind_str: &str, #[case] expected_kind: GitHubKind, #[case] url: &str) { + let kind: GitHubKind = serde_yaml::from_str(&format!("\"{}\"", kind_str)).unwrap(); + + assert_eq!(kind, expected_kind); + assert_eq!(kind.as_str(), kind_str); + assert_eq!(kind.api_endpoint(), url); + } } diff --git a/src/main.rs b/src/main.rs index 3b99f6d..d10e288 100644 --- a/src/main.rs +++ b/src/main.rs @@ -24,6 +24,7 @@ pub use entities::BackupEntity; pub use filter::{Filter, FilterValue, Filterable}; pub use policy::BackupPolicy; pub use sources::BackupSource; +use crate::helpers::github::GitHubKind; static CANCEL: AtomicBool = AtomicBool::new(false); @@ -48,7 +49,12 @@ async fn run(args: Args) -> Result<(), Error> { let config = config::Config::try_from(&args)?; let github_repo = - pairing::Pairing::new(sources::GitHubRepoSource::default(), engines::GitEngine) + pairing::Pairing::new(sources::GitHubRepoSource::repo(), engines::GitEngine) + .with_dry_run(args.dry_run) + .with_concurrency_limit(args.concurrency); + + let github_star = + pairing::Pairing::new(sources::GitHubRepoSource::star(), engines::GitEngine) .with_dry_run(args.dry_run) .with_concurrency_limit(args.concurrency); @@ -72,7 +78,7 @@ async fn run(args: Args) -> Result<(), Error> { let _policy_span = tracing::info_span!("backup.policy", policy = %policy).entered(); match policy.kind.as_str() { - "github/repo" => { + k if k == GitHubKind::Repo.as_str() => { info!("Backing up repositories for {}", &policy); let stream = github_repo.run(policy, &CANCEL); @@ -91,7 +97,26 @@ async fn run(args: Args) -> Result<(), Error> { } } } - "github/release" => { + k if k == GitHubKind::Star.as_str() => { + info!("Backing up repositories for {}", &policy); + + let stream = github_star.run(policy, &CANCEL); + tokio::pin!(stream); + while let Some(result) = stream.next().await { + match result { + Ok((entity, BackupState::Skipped)) => { + debug!(" - {} ({})", entity, BackupState::Skipped); + } + Ok((entity, state)) => { + info!(" - {} ({})", entity, state); + } + Err(e) => { + warn!("Error: {}", e); + } + } + } + } + k if k == GitHubKind::Release.as_str() => { info!("Backing up release artifacts for {}", &policy); let stream = github_release.run(policy, &CANCEL); diff --git a/src/sources/github_releases.rs b/src/sources/github_releases.rs index 6b65212..9ec807b 100644 --- a/src/sources/github_releases.rs +++ b/src/sources/github_releases.rs @@ -6,7 +6,7 @@ use crate::{ entities::{Credentials, HttpFile}, errors::{self}, helpers::{ - github::{GitHubRelease, GitHubRepo}, + github::{GitHubRelease, GitHubRepo, GitHubKind}, GitHubClient, }, policy::BackupPolicy, @@ -27,7 +27,7 @@ impl GitHubReleasesSource { impl BackupSource for GitHubReleasesSource { fn kind(&self) -> &str { - "github/release" + GitHubKind::Release.as_str() } fn validate(&self, policy: &BackupPolicy) -> Result<(), crate::Error> { @@ -58,13 +58,14 @@ impl BackupSource for GitHubReleasesSource { cancel: &'a AtomicBool, ) -> impl Stream> + 'a { let url = format!( - "{}/{}/repos", - policy - .properties - .get("api_url") - .unwrap_or(&"https://api.github.com".to_string()) - .trim_end_matches('/'), - &policy.from.trim_matches('/') + "{}/{}/{}", + policy + .properties + .get("api_url") + .unwrap_or(&"https://api.github.com".to_string()) + .trim_end_matches('/'), + &policy.from.trim_matches('/'), + GitHubKind::Release.api_endpoint() ); async_stream::stream! { diff --git a/src/sources/github_repo.rs b/src/sources/github_repo.rs index 2d63d2e..422faff 100644 --- a/src/sources/github_repo.rs +++ b/src/sources/github_repo.rs @@ -5,19 +5,19 @@ use tokio_stream::Stream; use crate::{ entities::GitRepo, errors::{self}, - helpers::{github::GitHubRepo, GitHubClient}, + helpers::{github::GitHubRepo, GitHubClient, github::GitHubKind}, policy::BackupPolicy, BackupSource, }; -#[derive(Clone, Default)] pub struct GitHubRepoSource { client: GitHubClient, + kind: GitHubKind, } impl BackupSource for GitHubRepoSource { fn kind(&self) -> &str { - "github/repo" + self.kind.as_str() } fn validate(&self, policy: &BackupPolicy) -> Result<(), crate::Error> { @@ -38,6 +38,11 @@ impl BackupSource for GitHubRepoSource { "Please specify either 'users/' or 'orgs/' as your target.", )), + t if t.starts_with("orgs/") && self.kind == GitHubKind::Star => Err(errors::user( + &format!("The target field '{target}' specifies an org which is not support for kind 'github/star'."), + "Please specify either 'users/' as your target when using 'github/star' as kind.", + )), + _ => Ok(()), } } @@ -48,13 +53,14 @@ impl BackupSource for GitHubRepoSource { cancel: &'a AtomicBool, ) -> impl Stream> + 'a { let url = format!( - "{}/{}/repos", + "{}/{}/{}", policy .properties .get("api_url") .unwrap_or(&"https://api.github.com".to_string()) .trim_end_matches('/'), - &policy.from.trim_matches('/') + &policy.from.trim_matches('/'), + self.kind.api_endpoint() ); async_stream::try_stream! { @@ -70,8 +76,25 @@ impl BackupSource for GitHubRepoSource { impl GitHubRepoSource { #[allow(dead_code)] - pub fn with_client(client: GitHubClient) -> Self { - GitHubRepoSource { client } + pub fn with_client(client: GitHubClient, kind: GitHubKind) -> Self { + GitHubRepoSource { + client: client, + kind: kind, + } + } + + pub fn repo() -> Self { + GitHubRepoSource { + client: GitHubClient::default(), + kind: GitHubKind::Repo, + } + } + + pub fn star() -> Self { + GitHubRepoSource { + client: GitHubClient::default(), + kind: GitHubKind::Star, + } } } @@ -81,15 +104,20 @@ mod tests { use rstest::rstest; - use crate::{BackupPolicy, BackupSource}; + use crate::{helpers::github::GitHubKind, BackupPolicy, BackupSource}; use super::GitHubRepoSource; static CANCEL: AtomicBool = AtomicBool::new(false); #[test] - fn check_name() { - assert_eq!(GitHubRepoSource::default().kind(), "github/repo"); + fn check_name_repo() { + assert_eq!(GitHubRepoSource::repo().kind(), GitHubKind::Repo.as_str()); + } + + #[test] + fn check_name_star() { + assert_eq!(GitHubRepoSource::star().kind(), GitHubKind::Star.as_str()); } #[rstest] @@ -98,8 +126,8 @@ mod tests { #[case("notheotherben", false)] #[case("sierrasoftworks/github-backup", false)] #[case("users/notheotherben/repos", false)] - fn validation(#[case] from: &str, #[case] success: bool) { - let source = GitHubRepoSource::default(); + fn validation_repo(#[case] from: &str, #[case] success: bool) { + let source = GitHubRepoSource::repo(); let policy = serde_yaml::from_str(&format!( r#" @@ -119,13 +147,36 @@ mod tests { } #[rstest] - #[case("users/notheotherben")] + #[case("users/notheotherben", true)] + #[case("orgs/sierrasoftworks", false)] + fn validation_stars(#[case] from: &str, #[case] success: bool) { + let source = GitHubRepoSource::star(); + + let policy = serde_yaml::from_str(&format!( + r#" + kind: github/star + from: {} + to: /tmp + "#, + from + )) + .expect("parse policy"); + + if success { + source.validate(&policy).expect("validation to succeed"); + } else { + source.validate(&policy).expect_err("validation to fail"); + } + } + + #[rstest] + #[case("users/cedi")] #[tokio::test] #[cfg_attr(feature = "pure_tests", ignore)] async fn get_repos(#[case] target: &str) { use tokio_stream::StreamExt; - let source = GitHubRepoSource::default(); + let source = GitHubRepoSource::repo(); let policy: BackupPolicy = serde_yaml::from_str(&format!( r#" @@ -150,4 +201,37 @@ mod tests { println!("{}", repo.expect("Failed to load repo")); } } + + #[rstest] + #[case("users/cedi")] + #[tokio::test] + #[cfg_attr(feature = "pure_tests", ignore)] + async fn get_stars(#[case] target: &str) { + use tokio_stream::StreamExt; + + let source = GitHubRepoSource::star(); + + let policy: BackupPolicy = serde_yaml::from_str(&format!( + r#" + kind: github/star + from: {} + to: /tmp + credentials: {} + "#, + target, + std::env::var("GITHUB_TOKEN") + .map(|t| format!("!Token {t}")) + .unwrap_or_else(|_| "!None".to_string()) + )) + .unwrap(); + + println!("Using credentials: {}", policy.credentials); + + let stream = source.load(&policy, &CANCEL); + tokio::pin!(stream); + + while let Some(repo) = stream.next().await { + println!("{}", repo.expect("Failed to load repo")); + } + } } From 77fa6dad25693f45c3bc7be439e2343f24977688 Mon Sep 17 00:00:00 2001 From: cedi Date: Thu, 28 Nov 2024 21:40:59 +0100 Subject: [PATCH 2/6] Update src/sources/github_repo.rs --- src/sources/github_repo.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sources/github_repo.rs b/src/sources/github_repo.rs index 422faff..333464c 100644 --- a/src/sources/github_repo.rs +++ b/src/sources/github_repo.rs @@ -203,7 +203,7 @@ mod tests { } #[rstest] - #[case("users/cedi")] + #[case("users/notheotherben")] #[tokio::test] #[cfg_attr(feature = "pure_tests", ignore)] async fn get_stars(#[case] target: &str) { From cf1e3bdcc6c2eaba00d92c0eefede3fcd83d809e Mon Sep 17 00:00:00 2001 From: cedi Date: Thu, 28 Nov 2024 21:41:04 +0100 Subject: [PATCH 3/6] Update src/sources/github_repo.rs --- src/sources/github_repo.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sources/github_repo.rs b/src/sources/github_repo.rs index 333464c..2630e94 100644 --- a/src/sources/github_repo.rs +++ b/src/sources/github_repo.rs @@ -170,7 +170,7 @@ mod tests { } #[rstest] - #[case("users/cedi")] + #[case("users/notheotherben")] #[tokio::test] #[cfg_attr(feature = "pure_tests", ignore)] async fn get_repos(#[case] target: &str) { From bdf775e4ff0d225db0e9672252835f6509b6a430 Mon Sep 17 00:00:00 2001 From: Cedric Kienzler Date: Thu, 28 Nov 2024 21:44:18 +0100 Subject: [PATCH 4/6] fix(source/github_repo.rs): restore #[derive(Clone)] Accidentally removed `#[derive(Clone)]` when the intention was to only remove `#[derive(Default)]` to remove the possibility to use `GitHubRepoSource::default()` --- src/helpers/github.rs | 3 +-- src/sources/github_repo.rs | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/helpers/github.rs b/src/helpers/github.rs index baa7702..842a26b 100644 --- a/src/helpers/github.rs +++ b/src/helpers/github.rs @@ -569,8 +569,7 @@ impl MetadataSource for GitHubReleaseAsset { } #[allow(dead_code)] -#[derive(PartialEq)] -#[derive(Debug)] +#[derive(PartialEq, Debug, Clone)] pub enum GitHubKind { Repo, Star, diff --git a/src/sources/github_repo.rs b/src/sources/github_repo.rs index 2630e94..db4627a 100644 --- a/src/sources/github_repo.rs +++ b/src/sources/github_repo.rs @@ -10,6 +10,7 @@ use crate::{ BackupSource, }; +#[derive(Clone)] pub struct GitHubRepoSource { client: GitHubClient, kind: GitHubKind, From d216b100bd4df26fbfd176b499813b0b85cd26a4 Mon Sep 17 00:00:00 2001 From: cedi Date: Fri, 29 Nov 2024 12:44:07 +0100 Subject: [PATCH 5/6] Apply suggestions from code review Co-authored-by: Benjamin Pannell <1760260+notheotherben@users.noreply.github.com> --- src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index d10e288..db030cd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -98,7 +98,7 @@ async fn run(args: Args) -> Result<(), Error> { } } k if k == GitHubKind::Star.as_str() => { - info!("Backing up repositories for {}", &policy); + info!("Backing up starred repositories for {}", &policy); let stream = github_star.run(policy, &CANCEL); tokio::pin!(stream); From abbc79fb1ba9a31cc040045a96680b09cc59a88e Mon Sep 17 00:00:00 2001 From: Cedric Kienzler Date: Fri, 29 Nov 2024 12:50:38 +0100 Subject: [PATCH 6/6] fix(GitHubKind): do no reimplement (de)serialization, relying on serde instead --- src/helpers/github.rs | 29 ++++------------------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/src/helpers/github.rs b/src/helpers/github.rs index 842a26b..b504735 100644 --- a/src/helpers/github.rs +++ b/src/helpers/github.rs @@ -569,10 +569,13 @@ impl MetadataSource for GitHubReleaseAsset { } #[allow(dead_code)] -#[derive(PartialEq, Debug, Clone)] +#[derive(PartialEq, Debug, Clone, serde::Serialize, serde::Deserialize)] pub enum GitHubKind { + #[serde(rename="github/repo")] Repo, + #[serde(rename="github/star")] Star, + #[serde(rename="github/release")] Release, } @@ -594,30 +597,6 @@ impl GitHubKind { } } -impl serde::Serialize for GitHubKind { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.serialize_str(self.as_str()) - } -} - -impl<'de> serde::Deserialize<'de> for GitHubKind { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let s: &str = serde::Deserialize::deserialize(deserializer)?; - match s { - "github/repo" => Ok(GitHubKind::Repo), - "github/star" => Ok(GitHubKind::Star), - "github/release" => Ok(GitHubKind::Release), - _ => Err(serde::de::Error::custom(format!("Invalid kind: {}", s))), - } - } -} - #[cfg(test)] mod tests { use std::path::PathBuf;