From ee617ea41313e740453d8c1605f229788c9093d0 Mon Sep 17 00:00:00 2001 From: Benjamin Pannell Date: Sun, 8 Dec 2024 23:11:29 +0000 Subject: [PATCH] feat: Add support for backing up repositories owned by the current user --- README.md | 8 +++++- examples/config.yaml | 17 ++++++++++++ src/helpers/github.rs | 26 +++++++++--------- src/main.rs | 8 +++--- src/sources/github_releases.rs | 9 +++--- src/sources/github_repo.rs | 50 ++++++++++++++++++++++------------ src/telemetry/mod.rs | 6 ++-- 7 files changed, 80 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 54cef99..589a877 100644 --- a/README.md +++ b/README.md @@ -40,11 +40,17 @@ schedule: "0 * * * *" backups: - kind: github/repo - from: users/my-user + from: user # The user associated with the provided credentials to: /backups/personal credentials: !UsernamePassword: username: "" password: "" + properties: + query: "affiliation=owner" # Additional query parameters to pass to GitHub when fetching repositories + - kind: github/repo + from: "users/another-user" + to: /backups/friend + credentials: !Token "your_github_token" - kind: github/repo from: "orgs/my-org" to: /backups/work diff --git a/examples/config.yaml b/examples/config.yaml index f0cf3a6..f5fea37 100644 --- a/examples/config.yaml +++ b/examples/config.yaml @@ -1,14 +1,31 @@ schedule: "0 * * * *" backups: + # Backup all the repositories that the provided credentials have access to + - kind: github/repo + from: user + to: /backup/github + credentials: !Token your_access_token + + # Backup the repository from "notheotherben" called "nix-env" - kind: github/repo from: users/notheotherben + to: /backup/github filter: repo.name == "nix-env" + + # Backup public, non-forked, repositories called "git-tool" or "grey" from the "SierraSoftworks" organization - kind: github/repo from: orgs/SierraSoftworks + to: /backup/github filter: repo.public && !repo.fork && repo.name in ["git-tool", "grey"] + + # Backup production non-source releases from the "SierraSoftworks" organization - kind: github/release from: orgs/SierraSoftworks + to: /backup/github filter: repo.public && !release.prerelease && !artifact.source-code + + # Backup all repositories that the user `notheotherben` has starred - kind: github/star from: users/notheotherben + to: /backup/github diff --git a/src/helpers/github.rs b/src/helpers/github.rs index 6b38a8a..0e27300 100644 --- a/src/helpers/github.rs +++ b/src/helpers/github.rs @@ -573,7 +573,7 @@ impl MetadataSource for GitHubReleaseAsset { #[allow(dead_code)] #[derive(PartialEq, Debug, Clone, serde::Serialize, serde::Deserialize)] -pub enum GitHubKind { +pub enum GitHubArtifactKind { #[serde(rename = "github/repo")] Repo, #[serde(rename = "github/star")] @@ -582,20 +582,20 @@ pub enum GitHubKind { Release, } -impl GitHubKind { +impl GitHubArtifactKind { pub fn as_str(&self) -> &'static str { match self { - GitHubKind::Repo => "github/repo", - GitHubKind::Star => "github/star", - GitHubKind::Release => "github/release", + GitHubArtifactKind::Repo => "github/repo", + GitHubArtifactKind::Star => "github/star", + GitHubArtifactKind::Release => "github/release", } } pub fn api_endpoint(&self) -> &'static str { match self { - GitHubKind::Repo => "repos", - GitHubKind::Star => "starred", - GitHubKind::Release => "repos", + GitHubArtifactKind::Repo => "repos", + GitHubArtifactKind::Star => "starred", + GitHubArtifactKind::Release => "repos", } } } @@ -720,15 +720,15 @@ mod tests { } #[rstest] - #[case("github/repo", GitHubKind::Repo, "repos")] - #[case("github/star", GitHubKind::Star, "starred")] - #[case("github/release", GitHubKind::Release, "repos")] + #[case("github/repo", GitHubArtifactKind::Repo, "repos")] + #[case("github/star", GitHubArtifactKind::Star, "starred")] + #[case("github/release", GitHubArtifactKind::Release, "repos")] fn test_deserialize_gh_repo_kind( #[case] kind_str: &str, - #[case] expected_kind: GitHubKind, + #[case] expected_kind: GitHubArtifactKind, #[case] url: &str, ) { - let kind: GitHubKind = serde_yaml::from_str(&format!("\"{}\"", kind_str)).unwrap(); + let kind: GitHubArtifactKind = serde_yaml::from_str(&format!("\"{}\"", kind_str)).unwrap(); assert_eq!(kind, expected_kind); assert_eq!(kind.as_str(), kind_str); diff --git a/src/main.rs b/src/main.rs index 166e531..aa77a79 100644 --- a/src/main.rs +++ b/src/main.rs @@ -20,7 +20,7 @@ mod policy; mod sources; mod telemetry; -use crate::helpers::github::GitHubKind; +use crate::helpers::github::GitHubArtifactKind; pub use entities::BackupEntity; pub use filter::{Filter, FilterValue, Filterable}; pub use policy::BackupPolicy; @@ -76,19 +76,19 @@ async fn run(args: Args) -> Result<(), Error> { let _policy_span = tracing::info_span!("backup.policy", policy = %policy).entered(); match policy.kind.as_str() { - k if k == GitHubKind::Repo.as_str() => { + k if k == GitHubArtifactKind::Repo.as_str() => { info!("Backing up repositories for {}", &policy); github_repo .run(policy, &LoggingPairingHandler, &CANCEL) .await; } - k if k == GitHubKind::Star.as_str() => { + k if k == GitHubArtifactKind::Star.as_str() => { info!("Backing up starred repositories for {}", &policy); github_star .run(policy, &LoggingPairingHandler, &CANCEL) .await; } - k if k == GitHubKind::Release.as_str() => { + k if k == GitHubArtifactKind::Release.as_str() => { info!("Backing up release artifacts for {}", &policy); github_release .run(policy, &LoggingPairingHandler, &CANCEL) diff --git a/src/sources/github_releases.rs b/src/sources/github_releases.rs index bddc505..64c7c87 100644 --- a/src/sources/github_releases.rs +++ b/src/sources/github_releases.rs @@ -6,7 +6,7 @@ use crate::{ entities::{Credentials, HttpFile}, errors::{self}, helpers::{ - github::{GitHubKind, GitHubRelease, GitHubRepo}, + github::{GitHubArtifactKind, GitHubRelease, GitHubRepo}, GitHubClient, }, policy::BackupPolicy, @@ -27,7 +27,7 @@ impl GitHubReleasesSource { impl BackupSource for GitHubReleasesSource { fn kind(&self) -> &str { - GitHubKind::Release.as_str() + GitHubArtifactKind::Release.as_str() } fn validate(&self, policy: &BackupPolicy) -> Result<(), crate::Error> { @@ -58,14 +58,15 @@ impl BackupSource for GitHubReleasesSource { cancel: &'a AtomicBool, ) -> impl Stream> + 'a { let url = format!( - "{}/{}/{}", + "{}/{}/{}?{}", policy .properties .get("api_url") .unwrap_or(&"https://api.github.com".to_string()) .trim_end_matches('/'), &policy.from.trim_matches('/'), - GitHubKind::Release.api_endpoint() + GitHubArtifactKind::Release.api_endpoint(), + policy.properties.get("query").unwrap_or(&"".to_string()) ); async_stream::stream! { diff --git a/src/sources/github_repo.rs b/src/sources/github_repo.rs index f1c2d7a..5baa6f2 100644 --- a/src/sources/github_repo.rs +++ b/src/sources/github_repo.rs @@ -5,7 +5,7 @@ use tokio_stream::Stream; use crate::{ entities::GitRepo, errors::{self}, - helpers::{github::GitHubKind, github::GitHubRepo, GitHubClient}, + helpers::{github::GitHubArtifactKind, github::GitHubRepo, GitHubClient}, policy::BackupPolicy, BackupSource, }; @@ -13,12 +13,12 @@ use crate::{ #[derive(Clone)] pub struct GitHubRepoSource { client: GitHubClient, - kind: GitHubKind, + artifact_kind: GitHubArtifactKind, } impl BackupSource for GitHubRepoSource { fn kind(&self) -> &str { - self.kind.as_str() + self.artifact_kind.as_str() } fn validate(&self, policy: &BackupPolicy) -> Result<(), crate::Error> { @@ -34,17 +34,19 @@ impl BackupSource for GitHubRepoSource { "Please provide a target field in the policy using the format 'users/' or 'orgs/'.", )), - t if !t.starts_with("users/") && !t.starts_with("orgs/") => Err(errors::user( - &format!("The target field '{target}' does not include a valid user or org specifier."), - "Please specify either 'users/' or 'orgs/' as your target.", - )), + t if t == "user" => Ok(()), + t if t.starts_with("users/") => Ok(()), - t if t.starts_with("orgs/") && self.kind == GitHubKind::Star => Err(errors::user( + t if t.starts_with("orgs/") && self.artifact_kind == GitHubArtifactKind::Star => Err(errors::user( &format!("The target field '{target}' specifies an org which is not support for kind 'github/star'."), "Please specify either 'users/' as your target when using 'github/star' as kind.", )), + t if t.starts_with("orgs/") => Ok(()), - _ => Ok(()), + _ => Err(errors::user( + &format!("The target field '{target}' does not include a valid user or org specifier."), + "Please specify either 'user', 'users/' or 'orgs/' as your target.", + )), } } @@ -54,14 +56,15 @@ impl BackupSource for GitHubRepoSource { cancel: &'a AtomicBool, ) -> impl Stream> + 'a { let url = format!( - "{}/{}/{}", + "{}/{}/{}?{}", policy .properties .get("api_url") .unwrap_or(&"https://api.github.com".to_string()) .trim_end_matches('/'), &policy.from.trim_matches('/'), - self.kind.api_endpoint() + self.artifact_kind.api_endpoint(), + policy.properties.get("query").unwrap_or(&"".to_string()) ); async_stream::try_stream! { @@ -77,21 +80,24 @@ impl BackupSource for GitHubRepoSource { impl GitHubRepoSource { #[allow(dead_code)] - pub fn with_client(client: GitHubClient, kind: GitHubKind) -> Self { - GitHubRepoSource { client, kind } + pub fn with_client(client: GitHubClient, kind: GitHubArtifactKind) -> Self { + GitHubRepoSource { + client, + artifact_kind: kind, + } } pub fn repo() -> Self { GitHubRepoSource { client: GitHubClient::default(), - kind: GitHubKind::Repo, + artifact_kind: GitHubArtifactKind::Repo, } } pub fn star() -> Self { GitHubRepoSource { client: GitHubClient::default(), - kind: GitHubKind::Star, + artifact_kind: GitHubArtifactKind::Star, } } } @@ -102,7 +108,7 @@ mod tests { use rstest::rstest; - use crate::{helpers::github::GitHubKind, BackupPolicy, BackupSource}; + use crate::{helpers::github::GitHubArtifactKind, BackupPolicy, BackupSource}; use super::GitHubRepoSource; @@ -110,15 +116,22 @@ mod tests { #[test] fn check_name_repo() { - assert_eq!(GitHubRepoSource::repo().kind(), GitHubKind::Repo.as_str()); + assert_eq!( + GitHubRepoSource::repo().kind(), + GitHubArtifactKind::Repo.as_str() + ); } #[test] fn check_name_star() { - assert_eq!(GitHubRepoSource::star().kind(), GitHubKind::Star.as_str()); + assert_eq!( + GitHubRepoSource::star().kind(), + GitHubArtifactKind::Star.as_str() + ); } #[rstest] + #[case("user", true)] #[case("users/notheotherben", true)] #[case("orgs/sierrasoftworks", true)] #[case("notheotherben", false)] @@ -145,6 +158,7 @@ mod tests { } #[rstest] + #[case("user", true)] #[case("users/notheotherben", true)] #[case("orgs/sierrasoftworks", false)] fn validation_stars(#[case] from: &str, #[case] success: bool) { diff --git a/src/telemetry/mod.rs b/src/telemetry/mod.rs index 1e77a27..2901c05 100644 --- a/src/telemetry/mod.rs +++ b/src/telemetry/mod.rs @@ -5,8 +5,6 @@ pub use traced_stream::*; use tracing_batteries::*; pub fn setup() -> Session { - Session::new("github-backup", version!()).with_battery( - OpenTelemetry::new("https://api.honeycomb.io") - .with_protocol(OpenTelemetryProtocol::HttpJson), - ) + Session::new("github-backup", version!()) + .with_battery(OpenTelemetry::new("").with_protocol(OpenTelemetryProtocol::HttpJson)) }