Merge pull request #106 from cedi/main

feat(github/star): Add support for backing up starred GitHub repos

examples/config.yaml

@@ -10,3 +10,5 @@ backups:
   - kind: github/release
     from: orgs/SierraSoftworks
     filter: repo.public && !release.prerelease && !artifact.source-code
+  - kind: github/star
+    from: users/notheotherben

src/helpers/github.rs

@@ -568,6 +568,35 @@ impl MetadataSource for GitHubReleaseAsset {
     }
 }
 
+#[allow(dead_code)]
+#[derive(PartialEq, Debug, Clone, serde::Serialize, serde::Deserialize)]
+pub enum GitHubKind {
+    #[serde(rename="github/repo")]
+    Repo,
+    #[serde(rename="github/star")]
+    Star,
+    #[serde(rename="github/release")]
+    Release,
+}
+
+impl GitHubKind {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            GitHubKind::Repo => "github/repo",
+            GitHubKind::Star => "github/star",
+            GitHubKind::Release => "github/release",
+        }
+    }
+
+    pub fn api_endpoint(&self) -> &'static str {
+        match self {
+            GitHubKind::Repo => "repos",
+            GitHubKind::Star => "starred",
+            GitHubKind::Release => "repos",
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use std::path::PathBuf;
@@ -686,4 +715,16 @@ mod tests {
             })
             .unwrap_or(Credentials::None)
     }
+
+    #[rstest]
+    #[case("github/repo", GitHubKind::Repo, "repos",)]
+    #[case("github/star", GitHubKind::Star, "starred",)]
+    #[case("github/release", GitHubKind::Release, "repos",)]
+    fn test_deserialize_gh_repo_kind(#[case] kind_str: &str, #[case] expected_kind: GitHubKind, #[case] url: &str) {
+        let kind: GitHubKind = serde_yaml::from_str(&format!("\"{}\"", kind_str)).unwrap();
+
+        assert_eq!(kind, expected_kind);
+        assert_eq!(kind.as_str(), kind_str);
+        assert_eq!(kind.api_endpoint(), url);
+    }
 }

src/main.rs

@@ -24,6 +24,7 @@ pub use entities::BackupEntity;
 pub use filter::{Filter, FilterValue, Filterable};
 pub use policy::BackupPolicy;
 pub use sources::BackupSource;
+use crate::helpers::github::GitHubKind;
 
 static CANCEL: AtomicBool = AtomicBool::new(false);
 
@@ -48,7 +49,12 @@ async fn run(args: Args) -> Result<(), Error> {
     let config = config::Config::try_from(&args)?;
 
     let github_repo =
-        pairing::Pairing::new(sources::GitHubRepoSource::default(), engines::GitEngine)
+        pairing::Pairing::new(sources::GitHubRepoSource::repo(), engines::GitEngine)
+            .with_dry_run(args.dry_run)
+            .with_concurrency_limit(args.concurrency);
+
+    let github_star =
+        pairing::Pairing::new(sources::GitHubRepoSource::star(), engines::GitEngine)
             .with_dry_run(args.dry_run)
             .with_concurrency_limit(args.concurrency);
 
@@ -72,7 +78,7 @@ async fn run(args: Args) -> Result<(), Error> {
                 let _policy_span = tracing::info_span!("backup.policy", policy = %policy).entered();
 
                 match policy.kind.as_str() {
-                    "github/repo" => {
+                    k if k == GitHubKind::Repo.as_str() => {
                         info!("Backing up repositories for {}", &policy);
 
                         let stream = github_repo.run(policy, &CANCEL);
@@ -91,7 +97,26 @@ async fn run(args: Args) -> Result<(), Error> {
                             }
                         }
                     }
-                    "github/release" => {
+                    k if k == GitHubKind::Star.as_str() => {
+                        info!("Backing up starred repositories for {}", &policy);
+
+                        let stream = github_star.run(policy, &CANCEL);
+                        tokio::pin!(stream);
+                        while let Some(result) = stream.next().await {
+                            match result {
+                                Ok((entity, BackupState::Skipped)) => {
+                                    debug!(" - {} ({})", entity, BackupState::Skipped);
+                                }
+                                Ok((entity, state)) => {
+                                    info!(" - {} ({})", entity, state);
+                                }
+                                Err(e) => {
+                                    warn!("Error: {}", e);
+                                }
+                            }
+                        }
+                    }
+                    k if k == GitHubKind::Release.as_str()  => {
                         info!("Backing up release artifacts for {}", &policy);
 
                         let stream = github_release.run(policy, &CANCEL);

src/sources/github_releases.rs

@@ -6,7 +6,7 @@ use crate::{
     entities::{Credentials, HttpFile},
     errors::{self},
     helpers::{
-        github::{GitHubRelease, GitHubRepo},
+        github::{GitHubRelease, GitHubRepo, GitHubKind},
         GitHubClient,
     },
     policy::BackupPolicy,
@@ -27,7 +27,7 @@ impl GitHubReleasesSource {
 
 impl BackupSource<HttpFile> for GitHubReleasesSource {
     fn kind(&self) -> &str {
-        "github/release"
+      GitHubKind::Release.as_str()
     }
 
     fn validate(&self, policy: &BackupPolicy) -> Result<(), crate::Error> {
@@ -58,13 +58,14 @@ impl BackupSource<HttpFile> for GitHubReleasesSource {
         cancel: &'a AtomicBool,
     ) -> impl Stream<Item = Result<HttpFile, crate::Error>> + 'a {
         let url = format!(
-            "{}/{}/repos",
-            policy
-                .properties
-                .get("api_url")
-                .unwrap_or(&"https://api.github.com".to_string())
-                .trim_end_matches('/'),
-            &policy.from.trim_matches('/')
+          "{}/{}/{}",
+          policy
+              .properties
+              .get("api_url")
+              .unwrap_or(&"https://api.github.com".to_string())
+              .trim_end_matches('/'),
+          &policy.from.trim_matches('/'),
+          GitHubKind::Release.api_endpoint()
         );
 
         async_stream::stream! {

src/sources/github_repo.rs

@@ -5,19 +5,20 @@ use tokio_stream::Stream;
 use crate::{
     entities::GitRepo,
     errors::{self},
-    helpers::{github::GitHubRepo, GitHubClient},
+    helpers::{github::GitHubRepo, GitHubClient, github::GitHubKind},
     policy::BackupPolicy,
     BackupSource,
 };
 
-#[derive(Clone, Default)]
+#[derive(Clone)]
 pub struct GitHubRepoSource {
     client: GitHubClient,
+    kind: GitHubKind,
 }
 
 impl BackupSource<GitRepo> for GitHubRepoSource {
     fn kind(&self) -> &str {
-        "github/repo"
+        self.kind.as_str()
     }
 
     fn validate(&self, policy: &BackupPolicy) -> Result<(), crate::Error> {
@@ -38,6 +39,11 @@ impl BackupSource<GitRepo> for GitHubRepoSource {
                 "Please specify either 'users/<username>' or 'orgs/<orgname>' as your target.",
             )),
 
+            t if t.starts_with("orgs/") && self.kind == GitHubKind::Star => Err(errors::user(
+                &format!("The target field '{target}' specifies an org which is not support for kind 'github/star'."),
+                "Please specify either 'users/<username>' as your target when using 'github/star' as kind.",
+            )),
+
             _ => Ok(()),
         }
     }
@@ -48,13 +54,14 @@ impl BackupSource<GitRepo> for GitHubRepoSource {
         cancel: &'a AtomicBool,
     ) -> impl Stream<Item = Result<GitRepo, errors::Error>> + 'a {
         let url = format!(
-            "{}/{}/repos",
+            "{}/{}/{}",
             policy
                 .properties
                 .get("api_url")
                 .unwrap_or(&"https://api.github.com".to_string())
                 .trim_end_matches('/'),
-            &policy.from.trim_matches('/')
+            &policy.from.trim_matches('/'),
+            self.kind.api_endpoint()
         );
 
         async_stream::try_stream! {
@@ -70,8 +77,25 @@ impl BackupSource<GitRepo> for GitHubRepoSource {
 
 impl GitHubRepoSource {
     #[allow(dead_code)]
-    pub fn with_client(client: GitHubClient) -> Self {
-        GitHubRepoSource { client }
+    pub fn with_client(client: GitHubClient, kind: GitHubKind) -> Self {
+        GitHubRepoSource {
+            client: client,
+            kind: kind,
+        }
+    }
+
+    pub fn repo() -> Self {
+        GitHubRepoSource {
+            client: GitHubClient::default(),
+            kind: GitHubKind::Repo,
+        }
+    }
+
+    pub fn star() -> Self {
+        GitHubRepoSource {
+            client: GitHubClient::default(),
+            kind: GitHubKind::Star,
+        }
     }
 }
 
@@ -81,15 +105,20 @@ mod tests {
 
     use rstest::rstest;
 
-    use crate::{BackupPolicy, BackupSource};
+    use crate::{helpers::github::GitHubKind, BackupPolicy, BackupSource};
 
     use super::GitHubRepoSource;
 
     static CANCEL: AtomicBool = AtomicBool::new(false);
 
     #[test]
-    fn check_name() {
-        assert_eq!(GitHubRepoSource::default().kind(), "github/repo");
+    fn check_name_repo() {
+        assert_eq!(GitHubRepoSource::repo().kind(), GitHubKind::Repo.as_str());
+    }
+
+    #[test]
+    fn check_name_star() {
+        assert_eq!(GitHubRepoSource::star().kind(), GitHubKind::Star.as_str());
     }
 
     #[rstest]
@@ -98,8 +127,8 @@ mod tests {
     #[case("notheotherben", false)]
     #[case("sierrasoftworks/github-backup", false)]
     #[case("users/notheotherben/repos", false)]
-    fn validation(#[case] from: &str, #[case] success: bool) {
-        let source = GitHubRepoSource::default();
+    fn validation_repo(#[case] from: &str, #[case] success: bool) {
+        let source = GitHubRepoSource::repo();
 
         let policy = serde_yaml::from_str(&format!(
             r#"
@@ -118,14 +147,37 @@ mod tests {
         }
     }
 
+    #[rstest]
+    #[case("users/notheotherben", true)]
+    #[case("orgs/sierrasoftworks", false)]
+    fn validation_stars(#[case] from: &str, #[case] success: bool) {
+        let source = GitHubRepoSource::star();
+
+        let policy = serde_yaml::from_str(&format!(
+            r#"
+            kind: github/star
+            from: {}
+            to: /tmp
+            "#,
+            from
+        ))
+        .expect("parse policy");
+
+        if success {
+            source.validate(&policy).expect("validation to succeed");
+        } else {
+            source.validate(&policy).expect_err("validation to fail");
+        }
+    }
+
     #[rstest]
     #[case("users/notheotherben")]
     #[tokio::test]
     #[cfg_attr(feature = "pure_tests", ignore)]
     async fn get_repos(#[case] target: &str) {
         use tokio_stream::StreamExt;
 
-        let source = GitHubRepoSource::default();
+        let source = GitHubRepoSource::repo();
 
         let policy: BackupPolicy = serde_yaml::from_str(&format!(
             r#"
@@ -150,4 +202,37 @@ mod tests {
             println!("{}", repo.expect("Failed to load repo"));
         }
     }
+
+    #[rstest]
+    #[case("users/notheotherben")]
+    #[tokio::test]
+    #[cfg_attr(feature = "pure_tests", ignore)]
+    async fn get_stars(#[case] target: &str) {
+        use tokio_stream::StreamExt;
+
+        let source = GitHubRepoSource::star();
+
+        let policy: BackupPolicy = serde_yaml::from_str(&format!(
+            r#"
+          kind: github/star
+          from: {}
+          to: /tmp
+          credentials: {}
+        "#,
+            target,
+            std::env::var("GITHUB_TOKEN")
+                .map(|t| format!("!Token {t}"))
+                .unwrap_or_else(|_| "!None".to_string())
+        ))
+        .unwrap();
+
+        println!("Using credentials: {}", policy.credentials);
+
+        let stream = source.load(&policy, &CANCEL);
+        tokio::pin!(stream);
+
+        while let Some(repo) = stream.next().await {
+            println!("{}", repo.expect("Failed to load repo"));
+        }
+    }
 }