diff --git a/examples/config.yaml b/examples/config.yaml index aeca02b..f0cf3a6 100644 --- a/examples/config.yaml +++ b/examples/config.yaml @@ -10,3 +10,5 @@ backups: - kind: github/release from: orgs/SierraSoftworks filter: repo.public && !release.prerelease && !artifact.source-code + - kind: github/star + from: users/notheotherben diff --git a/src/helpers/github.rs b/src/helpers/github.rs index 1dd9d3f..b504735 100644 --- a/src/helpers/github.rs +++ b/src/helpers/github.rs @@ -568,6 +568,35 @@ impl MetadataSource for GitHubReleaseAsset { } } +#[allow(dead_code)] +#[derive(PartialEq, Debug, Clone, serde::Serialize, serde::Deserialize)] +pub enum GitHubKind { + #[serde(rename="github/repo")] + Repo, + #[serde(rename="github/star")] + Star, + #[serde(rename="github/release")] + Release, +} + +impl GitHubKind { + pub fn as_str(&self) -> &'static str { + match self { + GitHubKind::Repo => "github/repo", + GitHubKind::Star => "github/star", + GitHubKind::Release => "github/release", + } + } + + pub fn api_endpoint(&self) -> &'static str { + match self { + GitHubKind::Repo => "repos", + GitHubKind::Star => "starred", + GitHubKind::Release => "repos", + } + } +} + #[cfg(test)] mod tests { use std::path::PathBuf; @@ -686,4 +715,16 @@ mod tests { }) .unwrap_or(Credentials::None) } + + #[rstest] + #[case("github/repo", GitHubKind::Repo, "repos",)] + #[case("github/star", GitHubKind::Star, "starred",)] + #[case("github/release", GitHubKind::Release, "repos",)] + fn test_deserialize_gh_repo_kind(#[case] kind_str: &str, #[case] expected_kind: GitHubKind, #[case] url: &str) { + let kind: GitHubKind = serde_yaml::from_str(&format!("\"{}\"", kind_str)).unwrap(); + + assert_eq!(kind, expected_kind); + assert_eq!(kind.as_str(), kind_str); + assert_eq!(kind.api_endpoint(), url); + } } diff --git a/src/main.rs b/src/main.rs index 3b99f6d..db030cd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -24,6 +24,7 @@ pub use entities::BackupEntity; pub use filter::{Filter, FilterValue, Filterable}; pub use policy::BackupPolicy; pub use sources::BackupSource; +use crate::helpers::github::GitHubKind; static CANCEL: AtomicBool = AtomicBool::new(false); @@ -48,7 +49,12 @@ async fn run(args: Args) -> Result<(), Error> { let config = config::Config::try_from(&args)?; let github_repo = - pairing::Pairing::new(sources::GitHubRepoSource::default(), engines::GitEngine) + pairing::Pairing::new(sources::GitHubRepoSource::repo(), engines::GitEngine) + .with_dry_run(args.dry_run) + .with_concurrency_limit(args.concurrency); + + let github_star = + pairing::Pairing::new(sources::GitHubRepoSource::star(), engines::GitEngine) .with_dry_run(args.dry_run) .with_concurrency_limit(args.concurrency); @@ -72,7 +78,7 @@ async fn run(args: Args) -> Result<(), Error> { let _policy_span = tracing::info_span!("backup.policy", policy = %policy).entered(); match policy.kind.as_str() { - "github/repo" => { + k if k == GitHubKind::Repo.as_str() => { info!("Backing up repositories for {}", &policy); let stream = github_repo.run(policy, &CANCEL); @@ -91,7 +97,26 @@ async fn run(args: Args) -> Result<(), Error> { } } } - "github/release" => { + k if k == GitHubKind::Star.as_str() => { + info!("Backing up starred repositories for {}", &policy); + + let stream = github_star.run(policy, &CANCEL); + tokio::pin!(stream); + while let Some(result) = stream.next().await { + match result { + Ok((entity, BackupState::Skipped)) => { + debug!(" - {} ({})", entity, BackupState::Skipped); + } + Ok((entity, state)) => { + info!(" - {} ({})", entity, state); + } + Err(e) => { + warn!("Error: {}", e); + } + } + } + } + k if k == GitHubKind::Release.as_str() => { info!("Backing up release artifacts for {}", &policy); let stream = github_release.run(policy, &CANCEL); diff --git a/src/sources/github_releases.rs b/src/sources/github_releases.rs index 6b65212..9ec807b 100644 --- a/src/sources/github_releases.rs +++ b/src/sources/github_releases.rs @@ -6,7 +6,7 @@ use crate::{ entities::{Credentials, HttpFile}, errors::{self}, helpers::{ - github::{GitHubRelease, GitHubRepo}, + github::{GitHubRelease, GitHubRepo, GitHubKind}, GitHubClient, }, policy::BackupPolicy, @@ -27,7 +27,7 @@ impl GitHubReleasesSource { impl BackupSource for GitHubReleasesSource { fn kind(&self) -> &str { - "github/release" + GitHubKind::Release.as_str() } fn validate(&self, policy: &BackupPolicy) -> Result<(), crate::Error> { @@ -58,13 +58,14 @@ impl BackupSource for GitHubReleasesSource { cancel: &'a AtomicBool, ) -> impl Stream> + 'a { let url = format!( - "{}/{}/repos", - policy - .properties - .get("api_url") - .unwrap_or(&"https://api.github.com".to_string()) - .trim_end_matches('/'), - &policy.from.trim_matches('/') + "{}/{}/{}", + policy + .properties + .get("api_url") + .unwrap_or(&"https://api.github.com".to_string()) + .trim_end_matches('/'), + &policy.from.trim_matches('/'), + GitHubKind::Release.api_endpoint() ); async_stream::stream! { diff --git a/src/sources/github_repo.rs b/src/sources/github_repo.rs index 2d63d2e..db4627a 100644 --- a/src/sources/github_repo.rs +++ b/src/sources/github_repo.rs @@ -5,19 +5,20 @@ use tokio_stream::Stream; use crate::{ entities::GitRepo, errors::{self}, - helpers::{github::GitHubRepo, GitHubClient}, + helpers::{github::GitHubRepo, GitHubClient, github::GitHubKind}, policy::BackupPolicy, BackupSource, }; -#[derive(Clone, Default)] +#[derive(Clone)] pub struct GitHubRepoSource { client: GitHubClient, + kind: GitHubKind, } impl BackupSource for GitHubRepoSource { fn kind(&self) -> &str { - "github/repo" + self.kind.as_str() } fn validate(&self, policy: &BackupPolicy) -> Result<(), crate::Error> { @@ -38,6 +39,11 @@ impl BackupSource for GitHubRepoSource { "Please specify either 'users/' or 'orgs/' as your target.", )), + t if t.starts_with("orgs/") && self.kind == GitHubKind::Star => Err(errors::user( + &format!("The target field '{target}' specifies an org which is not support for kind 'github/star'."), + "Please specify either 'users/' as your target when using 'github/star' as kind.", + )), + _ => Ok(()), } } @@ -48,13 +54,14 @@ impl BackupSource for GitHubRepoSource { cancel: &'a AtomicBool, ) -> impl Stream> + 'a { let url = format!( - "{}/{}/repos", + "{}/{}/{}", policy .properties .get("api_url") .unwrap_or(&"https://api.github.com".to_string()) .trim_end_matches('/'), - &policy.from.trim_matches('/') + &policy.from.trim_matches('/'), + self.kind.api_endpoint() ); async_stream::try_stream! { @@ -70,8 +77,25 @@ impl BackupSource for GitHubRepoSource { impl GitHubRepoSource { #[allow(dead_code)] - pub fn with_client(client: GitHubClient) -> Self { - GitHubRepoSource { client } + pub fn with_client(client: GitHubClient, kind: GitHubKind) -> Self { + GitHubRepoSource { + client: client, + kind: kind, + } + } + + pub fn repo() -> Self { + GitHubRepoSource { + client: GitHubClient::default(), + kind: GitHubKind::Repo, + } + } + + pub fn star() -> Self { + GitHubRepoSource { + client: GitHubClient::default(), + kind: GitHubKind::Star, + } } } @@ -81,15 +105,20 @@ mod tests { use rstest::rstest; - use crate::{BackupPolicy, BackupSource}; + use crate::{helpers::github::GitHubKind, BackupPolicy, BackupSource}; use super::GitHubRepoSource; static CANCEL: AtomicBool = AtomicBool::new(false); #[test] - fn check_name() { - assert_eq!(GitHubRepoSource::default().kind(), "github/repo"); + fn check_name_repo() { + assert_eq!(GitHubRepoSource::repo().kind(), GitHubKind::Repo.as_str()); + } + + #[test] + fn check_name_star() { + assert_eq!(GitHubRepoSource::star().kind(), GitHubKind::Star.as_str()); } #[rstest] @@ -98,8 +127,8 @@ mod tests { #[case("notheotherben", false)] #[case("sierrasoftworks/github-backup", false)] #[case("users/notheotherben/repos", false)] - fn validation(#[case] from: &str, #[case] success: bool) { - let source = GitHubRepoSource::default(); + fn validation_repo(#[case] from: &str, #[case] success: bool) { + let source = GitHubRepoSource::repo(); let policy = serde_yaml::from_str(&format!( r#" @@ -118,6 +147,29 @@ mod tests { } } + #[rstest] + #[case("users/notheotherben", true)] + #[case("orgs/sierrasoftworks", false)] + fn validation_stars(#[case] from: &str, #[case] success: bool) { + let source = GitHubRepoSource::star(); + + let policy = serde_yaml::from_str(&format!( + r#" + kind: github/star + from: {} + to: /tmp + "#, + from + )) + .expect("parse policy"); + + if success { + source.validate(&policy).expect("validation to succeed"); + } else { + source.validate(&policy).expect_err("validation to fail"); + } + } + #[rstest] #[case("users/notheotherben")] #[tokio::test] @@ -125,7 +177,7 @@ mod tests { async fn get_repos(#[case] target: &str) { use tokio_stream::StreamExt; - let source = GitHubRepoSource::default(); + let source = GitHubRepoSource::repo(); let policy: BackupPolicy = serde_yaml::from_str(&format!( r#" @@ -150,4 +202,37 @@ mod tests { println!("{}", repo.expect("Failed to load repo")); } } + + #[rstest] + #[case("users/notheotherben")] + #[tokio::test] + #[cfg_attr(feature = "pure_tests", ignore)] + async fn get_stars(#[case] target: &str) { + use tokio_stream::StreamExt; + + let source = GitHubRepoSource::star(); + + let policy: BackupPolicy = serde_yaml::from_str(&format!( + r#" + kind: github/star + from: {} + to: /tmp + credentials: {} + "#, + target, + std::env::var("GITHUB_TOKEN") + .map(|t| format!("!Token {t}")) + .unwrap_or_else(|_| "!None".to_string()) + )) + .unwrap(); + + println!("Using credentials: {}", policy.credentials); + + let stream = source.load(&policy, &CANCEL); + tokio::pin!(stream); + + while let Some(repo) = stream.next().await { + println!("{}", repo.expect("Failed to load repo")); + } + } }