From e0a2de8af191d3b5cbb0403194488c88e3e06b4b Mon Sep 17 00:00:00 2001 From: Xeckt Date: Mon, 14 Oct 2024 15:27:11 +0100 Subject: [PATCH 01/24] amend git.rs to use new config instead of env, add debug derives to support new changes --- backend/src/db.rs | 2 +- backend/src/gh.rs | 2 +- backend/src/git.rs | 51 +++++++++++++++++++++++----------------------- 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/backend/src/db.rs b/backend/src/db.rs index fa31b4a..e203de2 100644 --- a/backend/src/db.rs +++ b/backend/src/db.rs @@ -42,7 +42,7 @@ pub struct GroupPermissions { } /// A wrapper around the sqlite database, and how consumers should interact with the database in any capacity. -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct Database { pool: SqlitePool, } diff --git a/backend/src/gh.rs b/backend/src/gh.rs index c157b52..8d663a4 100644 --- a/backend/src/gh.rs +++ b/backend/src/gh.rs @@ -59,7 +59,7 @@ impl Claims { } /// A wrapper around the github access token that automatically refreshes if the token has been invalidated -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct GithubAccessToken { expires_at: Arc>, token: Arc>, diff --git a/backend/src/git.rs b/backend/src/git.rs index a323ae3..a00c6da 100644 --- a/backend/src/git.rs +++ b/backend/src/git.rs @@ -8,13 +8,13 @@ use std::fs; use std::io::{Read, Write}; use std::path::Path; use std::{ - env, path::PathBuf, sync::{Arc, Mutex}, }; use tracing::{debug, info, warn}; +use crate::AppState; -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct Interface { repo: Arc>, /// The path to the documents folder, relative to the server executable. @@ -37,13 +37,11 @@ impl Interface { /// # Errors /// This function will return an error if any of the git initialization steps fail, or if /// the required environment variables are not set. - pub fn new() -> Result { + pub fn new(state: AppState) -> Result { + let config = Arc::clone(&state.config); let mut doc_path = PathBuf::from("repo"); - doc_path.push(env::var("DOC_PATH").unwrap_or_else(|_| { - warn!("The `DOC_PATH` environment variable was not set, defaulting to `docs/`"); - "docs".to_string() - })); - let repo = Self::load_repository("repo")?; + doc_path.push(&config.files.docs_path); + let repo = Self::load_repository("repo", state)?; Ok(Self { repo: Arc::new(Mutex::new(repo)), doc_path, @@ -126,11 +124,13 @@ impl Interface { #[tracing::instrument(skip_all)] pub fn put_doc + Copy + std::fmt::Debug>( &self, + state: AppState, path: P, new_doc: &str, message: &str, token: &str, ) -> Result<()> { + let config = Arc::clone(&state.config); let repo = self.repo.lock().unwrap(); let mut path_to_doc: PathBuf = PathBuf::from("."); path_to_doc.push(&self.doc_path); @@ -151,15 +151,13 @@ impl Interface { })?; let msg = format!("[Hyde]: {message}"); // Relative to the root of the repo, not the current dir, so typically `./docs` instead of `./repo/docs` - let mut relative_path = PathBuf::from( - env::var("DOC_PATH").wrap_err("The `DOC_PATH` environment variable was not set")?, - ); + let mut relative_path = PathBuf::from(&config.files.repo_url); // Standard practice is to stage commits by adding them to an index. relative_path.push(path); Self::git_add(&repo, relative_path)?; let commit_id = Self::git_commit(&repo, msg, None)?; debug!("New commit made with ID: {:?}", commit_id); - Self::git_push(&repo, token)?; + Self::git_push(&repo, token, state)?; info!( "Document {:?} edited and pushed to GitHub with message: {message:?}", path.as_ref() @@ -180,26 +178,26 @@ impl Interface { // it creates errors that note the destructor for other values failing because of it (tree) pub fn delete_doc + Copy>( &self, + state: AppState, path: P, message: &str, token: &str, ) -> Result<()> { + let config = Arc::clone(&state.config); let repo = self.repo.lock().unwrap(); let mut path_to_doc: PathBuf = PathBuf::new(); path_to_doc.push(&self.doc_path); path_to_doc.push(path); let msg = format!("[Hyde]: {message}"); // Relative to the root of the repo, not the current dir, so typically `./docs` instead of `./repo/docs` - let mut relative_path = PathBuf::from( - env::var("DOC_PATH").wrap_err("The `DOC_PATH` environment variable was not set")?, - ); + let mut relative_path = PathBuf::from(&config.files.docs_path); // Standard practice is to stage commits by adding them to an index. relative_path.push(path); fs::remove_file(&path_to_doc).wrap_err_with(|| format!("Failed to remove document the document at {path_to_doc:?}"))?; Self::git_add(&repo, ".")?; let commit_id = Self::git_commit(&repo, msg, None)?; debug!("New commit made with ID: {:?}", commit_id); - Self::git_push(&repo, token)?; + Self::git_push(&repo, token, state)?; drop(repo); info!( "Document {:?} removed and changes synced to Github with message: {message:?}", @@ -212,15 +210,15 @@ impl Interface { /// If the repository at the provided path exists, open it and fetch the latest changes from the `master` branch. /// If not, clone into the provided path. #[tracing::instrument] - fn load_repository + std::fmt::Debug>(path: P) -> Result { + fn load_repository + std::fmt::Debug>(path: P, state: AppState) -> Result { + let config = Arc::clone(&state.config); if let Ok(repo) = Repository::open("./repo") { Self::git_pull(&repo)?; info!("Existing repository detected, fetching latest changes..."); return Ok(repo); } - let repository_url = env::var("REPO_URL") - .wrap_err("The `REPO_URL` environment url was not set, this is required.")?; + let repository_url = &config.files.repo_url; let output_path = Path::new("./repo"); info!( "No repo detected, cloning {repository_url:?} into {:?}...", @@ -233,14 +231,14 @@ impl Interface { /// Completely clone and open a new repository, deleting the old one. #[tracing::instrument(skip_all)] - pub fn reclone(&self) -> Result<()> { + pub fn reclone(&self, state: AppState) -> Result<()> { // First clone a repo into `repo__tmp`, open that, swap out // TODO: nuke `repo__tmp` if it exists already - let repo_path = Path::new("./repo"); - let tmp_path = Path::new("./repo__tmp"); + let repo_path = Path::new("./repo"); // TODO: Possibly implement this path into new config? + let tmp_path = Path::new("./repo__tmp"); // TODO: Same here? + let config = Arc::clone(&state.config); info!("Re-cloning repository, temporary repo will be created at {tmp_path:?}"); - let repository_url = env::var("REPO_URL") - .wrap_err("The `REPO_URL` environment url was not set, this is required.")?; + let repository_url = &config.files.repo_url; let tmp_repo = Repository::clone(&repository_url, tmp_path)?; info!("Pointing changes to new temp repository"); let mut lock = self.repo.lock().unwrap(); @@ -322,8 +320,9 @@ impl Interface { /// /// `token` is a valid Github auth token. // TODO: stop hardcoding refspec and make it an argument. - fn git_push(repo: &Repository, token: &str) -> Result<()> { - let repository_url = env::var("REPO_URL").wrap_err("Repo url not set in env")?; + fn git_push(repo: &Repository, token: &str, state: AppState) -> Result<()> { + let config = Arc::clone(&state.config); + let repository_url = &config.files.repo_url; let authenticated_url = repository_url.replace("https://", &format!("https://x-access-token:{token}@")); repo.remote_set_pushurl("origin", Some(&authenticated_url))?; From c3c829e5dfbab389ac91f79c33de8ecefc8b2257 Mon Sep 17 00:00:00 2001 From: Xeckt Date: Mon, 14 Oct 2024 16:22:07 +0100 Subject: [PATCH 02/24] make changes to startup for new config --- backend/src/main.rs | 58 +++++++++++++++++---------------------------- 1 file changed, 22 insertions(+), 36 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index 30a9a12..76ed114 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -8,6 +8,7 @@ mod gh; pub mod git; mod handlers_prelude; pub mod perms; +mod hyde_config; use axum::{ extract::MatchedPath, @@ -34,6 +35,7 @@ use reqwest::{ Client, Method, }; use std::env::{self, current_exe}; +use std::sync::Arc; use std::time::Duration; #[cfg(target_family = "unix")] use tokio::signal::unix::{signal, SignalKind}; @@ -44,10 +46,12 @@ use tokio::task; use tower_http::cors::CorsLayer; use tower_http::trace::TraceLayer; use tower_http::{normalize_path::NormalizePathLayer, services::ServeDir}; +use crate::hyde_config::HydeConfig; /// Global app state passed to handlers by axum #[derive(Clone)] -struct AppState { +pub struct AppState { + pub config: Arc, git: git::Interface, oauth: BasicClient, reqwest_client: Client, @@ -84,8 +88,6 @@ async fn main() -> Result<()> { color_eyre::install()?; // Parse command line arguments let cli_args = Args::parse(); - // Read environment variables from dotenv file - let dotenv_path = "hyde-data/.env"; // Load in any config settings passed by cli for (key, value) in &cli_args.cfg { env::set_var(key, value); @@ -97,10 +99,6 @@ async fn main() -> Result<()> { .init(); debug!("Initialized logging"); - dotenvy::from_path(dotenv_path).unwrap_or_else(|_| { - warn!("Failed to read dotenv file located at {dotenv_path}, please ensure all config values are manually set"); - }); - if cfg!(debug_assertions) { info!("Server running in development mode, version v{}", env!("CARGO_PKG_VERSION")); } else { @@ -136,40 +134,28 @@ async fn main() -> Result<()> { /// Initialize an instance of [`AppState`] #[tracing::instrument] async fn init_state() -> Result { - let git = task::spawn(async { git::Interface::new() }); - let oauth = { - let client_id = env::var("OAUTH_CLIENT_ID").unwrap_or_else(|_| { - warn!("The `OAUTH_CLIENT_ID` environment variable is not set, oauth functionality will be broken"); - String::new() - }); - let client_secret = env::var("OAUTH_SECRET").unwrap_or_else(|_| { - warn!("The `OAUTH_SECRET` environment variable is not set, oauth functionality will be broken"); - String::new() - }); - // The oauth constructor does some url parsing on startup so these need valid urls - let auth_url = env::var("OAUTH_URL").unwrap_or_else(|_| { - warn!("The `OAUTH_URL` environment variable is not set, oauth functionality will be broken"); - String::from("https://example.com/") - }); - let token_url = env::var("OAUTH_TOKEN_URL").unwrap_or_else(|_| { - warn!("The `OAUTH_TOKEN_URL` environment variable is not set, oauth functionality will be broken"); - String::from("https://example.com/") - }); - BasicClient::new( - ClientId::new(client_id), - Some(ClientSecret::new(client_secret)), - AuthUrl::new(auth_url)?, - Some(TokenUrl::new(token_url)?), - ) - }; + let config = HydeConfig::load(); + let repo_url = config.files.repo_url.clone(); + let git = task::spawn(async { git::Interface::new(repo_url)}).await??; let reqwest_client = Client::new(); + + // We have to clone here, since the client will need to keep the values from the config. + let oauth = BasicClient::new( + ClientId::new(config.oauth.discord.client_id.clone()), + Some(ClientSecret::new(config.oauth.discord.secret.clone())), + AuthUrl::new(config.oauth.discord.url.clone())?, + Some(TokenUrl::new(config.oauth.discord.token_url.clone())?), + ); + Ok(AppState { - git: git.await??, + config, + git, oauth, reqwest_client, gh_credentials: GithubAccessToken::new(), db: Database::new().await?, }) + } /// Parse a single key-value pair for clap list parsing @@ -189,8 +175,8 @@ async fn start_server(state: AppState, cli_args: Args) -> Result<()> { // current_exe returns the path of the file, we need the dir the file is in frontend_dir.pop(); frontend_dir.push("web"); - let asset_path = env::var("ASSET_PATH") - .wrap_err("The `ASSET_PATH` environment variable was not set in the env")?; + let config = Arc::clone(&state.config); + let asset_path = &config.files.asset_path; // Initialize the handler and router let api_routes = Router::new() From e6fd60c41047e75249e4f70387c5cc81c7817f18 Mon Sep 17 00:00:00 2001 From: Xeckt Date: Mon, 14 Oct 2024 16:22:14 +0100 Subject: [PATCH 03/24] amend doc.rs for new config --- backend/src/handlers_prelude/doc.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/backend/src/handlers_prelude/doc.rs b/backend/src/handlers_prelude/doc.rs index dcef9da..c646a22 100644 --- a/backend/src/handlers_prelude/doc.rs +++ b/backend/src/handlers_prelude/doc.rs @@ -1,3 +1,4 @@ +use std::sync::Arc; use axum::{debug_handler, extract::{Query, State}, http::{HeaderMap, StatusCode}, Json, Router}; use axum::routing::get; use serde::{Deserialize, Serialize}; @@ -65,7 +66,7 @@ pub async fn put_doc_handler( ) .await?; - let gh_token = match &state.gh_credentials.get(&state.reqwest_client).await { + let gh_token = match &state.gh_credentials.get(&state.reqwest_client, Arc::clone(&state.config)).await { Ok(t) => t.clone(), Err(e) => { error!("Failed to authenticate with github for a put_doc request with error: {e:?}"); @@ -82,7 +83,7 @@ pub async fn put_doc_handler( match state .git - .put_doc(&body.path, &body.contents, &final_commit_message, &gh_token) + .put_doc(Arc::clone(&state.config), &body.path, &body.contents, &final_commit_message, &gh_token) { Ok(_) => Ok(StatusCode::CREATED), Err(e) => { @@ -107,8 +108,8 @@ pub async fn delete_doc_handler( ) .await?; - let gh_token = state.gh_credentials.get(&state.reqwest_client).await.unwrap(); - state.git.delete_doc(&query.path, &format!("{} deleted {}", author.username, query.path), &gh_token).map_err(eyre_to_axum_err)?; + let gh_token = state.gh_credentials.get(&state.reqwest_client, Arc::clone(&state.config)).await.unwrap(); + state.git.delete_doc(Arc::clone(&state.config), &query.path, &format!("{} deleted {}", author.username, query.path), &gh_token).map_err(eyre_to_axum_err)?; Ok(StatusCode::NO_CONTENT) } From ceca8bfa92e43bc7b34fe1cc28313adf259e2b45 Mon Sep 17 00:00:00 2001 From: Xeckt Date: Mon, 14 Oct 2024 16:22:23 +0100 Subject: [PATCH 04/24] amend gh.rs for new config --- backend/src/gh.rs | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/backend/src/gh.rs b/backend/src/gh.rs index 8d663a4..decd048 100644 --- a/backend/src/gh.rs +++ b/backend/src/gh.rs @@ -9,9 +9,12 @@ use serde::{Deserialize, Serialize}; use std::env; use std::fs::File; use std::io::Read; +use std::os::linux::raw::stat; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; use tokio::sync::Mutex; +use crate::AppState; +use crate::hyde_config::HydeConfig; /// In order to authenticate as a github app or generate an installation access token, you must generate a JSON Web Token (JWT). The JWT must contain predefined *claims*. /// @@ -43,11 +46,11 @@ struct Claims { } impl Claims { - pub fn new() -> Result { + pub fn new(config: Arc) -> Result { let current_time = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); let iat = current_time - 60; let exp = current_time + (60 * 5); - let iss = env::var("GH_CLIENT_ID").wrap_err("Failed to read the `GH_CLIENT_ID` env var")?; + let iss = config.oauth.github.client_id.clone(); Ok(Self { iat, @@ -76,12 +79,12 @@ impl GithubAccessToken { } /// Return the cached token if it's less than one hour old, or fetch a new token from the api, and return that, updating the cache - pub async fn get(&self, req_client: &Client) -> Result { + pub async fn get(&self, req_client: &Client, config: Arc) -> Result { let mut token_ref = self.token.lock().await; // Fetch a new token if more than 59 minutes have passed // Tokens expire after 1 hour, this is to account for clock drift if SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs() > (60 * 59) { - let api_response = get_access_token(req_client).await?; + let api_response = get_access_token(req_client, config).await?; *token_ref = api_response.0; let mut expires_ref = self.expires_at.lock().await; *expires_ref = api_response.1; @@ -99,12 +102,12 @@ struct AccessTokenResponse { /// Request a github installation access token using the provided reqwest client. /// The installation access token will expire after 1 hour. /// Returns the new token, and the time of expiration -async fn get_access_token(req_client: &Client) -> Result<(String, SystemTime)> { - let token = gen_jwt_token()?; +async fn get_access_token(req_client: &Client, config: Arc) -> Result<(String, SystemTime)> { + let token = gen_jwt_token(Arc::clone(&config))?; let response = req_client .post(format!( "https://api.github.com/app/installations/{}/access_tokens", - get_installation_id(req_client).await? + get_installation_id(req_client, Arc::clone(&config)).await? )) .bearer_auth(token) .header("Accept", "application/vnd.github+json") @@ -129,10 +132,10 @@ struct InstallationIdResponse { /// Fetch the Installation ID. This value is required for most API calls /// /// -async fn get_installation_id(req_client: &Client) -> Result { +async fn get_installation_id(req_client: &Client, config: Arc) -> Result { let response = req_client .get("https://api.github.com/app/installations") - .bearer_auth(gen_jwt_token()?) + .bearer_auth(gen_jwt_token(config)?) .header("User-Agent", "Hyde") // https://docs.github.com/en/rest/about-the-rest-api/api-versions?apiVersion=2022-11-28 .header("X-GitHub-Api-Version", "2022-11-28") @@ -151,14 +154,14 @@ async fn get_installation_id(req_client: &Client) -> Result { } /// Generate a new JWT token for use with github api interactions. -fn gen_jwt_token() -> Result { +fn gen_jwt_token(config: Arc) -> Result { let mut private_key_file = File::open("hyde-data/key.pem") .wrap_err("Failed to read private key from `hyde-data/key.pem`")?; let mut private_key = Vec::new(); private_key_file.read_to_end(&mut private_key)?; Ok(encode( &Header::new(Algorithm::RS256), - &Claims::new()?, + &Claims::new(config)?, &EncodingKey::from_rsa_pem(&private_key)?, )?) } From cc1d6bc4412df7b7deb8bea30e919162e56d88a8 Mon Sep 17 00:00:00 2001 From: Xeckt Date: Mon, 14 Oct 2024 16:22:29 +0100 Subject: [PATCH 05/24] amend git.rs for new config --- backend/src/git.rs | 47 ++++++++++++++++++++-------------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/backend/src/git.rs b/backend/src/git.rs index a00c6da..162fc97 100644 --- a/backend/src/git.rs +++ b/backend/src/git.rs @@ -2,7 +2,7 @@ use color_eyre::eyre::{bail, ContextCompat}; use color_eyre::{eyre::Context, Result}; -use git2::{AnnotatedCommit, FetchOptions, Oid, Repository, Signature, Status}; +use git2::{AnnotatedCommit, Config, FetchOptions, Oid, Repository, Signature, Status}; use serde::{Deserialize, Serialize}; use std::fs; use std::io::{Read, Write}; @@ -12,9 +12,9 @@ use std::{ sync::{Arc, Mutex}, }; use tracing::{debug, info, warn}; -use crate::AppState; +use crate::hyde_config::HydeConfig; -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct Interface { repo: Arc>, /// The path to the documents folder, relative to the server executable. @@ -37,11 +37,10 @@ impl Interface { /// # Errors /// This function will return an error if any of the git initialization steps fail, or if /// the required environment variables are not set. - pub fn new(state: AppState) -> Result { - let config = Arc::clone(&state.config); + pub fn new(repo_url: String) -> Result { let mut doc_path = PathBuf::from("repo"); - doc_path.push(&config.files.docs_path); - let repo = Self::load_repository("repo", state)?; + doc_path.push(&repo_url); + let repo = Self::load_repository("repo", repo_url)?; Ok(Self { repo: Arc::new(Mutex::new(repo)), doc_path, @@ -124,13 +123,13 @@ impl Interface { #[tracing::instrument(skip_all)] pub fn put_doc + Copy + std::fmt::Debug>( &self, - state: AppState, + config: Arc, path: P, new_doc: &str, message: &str, token: &str, ) -> Result<()> { - let config = Arc::clone(&state.config); + let config = Arc::clone(&config); let repo = self.repo.lock().unwrap(); let mut path_to_doc: PathBuf = PathBuf::from("."); path_to_doc.push(&self.doc_path); @@ -157,7 +156,7 @@ impl Interface { Self::git_add(&repo, relative_path)?; let commit_id = Self::git_commit(&repo, msg, None)?; debug!("New commit made with ID: {:?}", commit_id); - Self::git_push(&repo, token, state)?; + Self::git_push(&repo, token, &config.files.repo_url)?; info!( "Document {:?} edited and pushed to GitHub with message: {message:?}", path.as_ref() @@ -178,12 +177,12 @@ impl Interface { // it creates errors that note the destructor for other values failing because of it (tree) pub fn delete_doc + Copy>( &self, - state: AppState, + config: Arc, path: P, message: &str, token: &str, ) -> Result<()> { - let config = Arc::clone(&state.config); + let config = Arc::clone(&config); let repo = self.repo.lock().unwrap(); let mut path_to_doc: PathBuf = PathBuf::new(); path_to_doc.push(&self.doc_path); @@ -197,7 +196,7 @@ impl Interface { Self::git_add(&repo, ".")?; let commit_id = Self::git_commit(&repo, msg, None)?; debug!("New commit made with ID: {:?}", commit_id); - Self::git_push(&repo, token, state)?; + Self::git_push(&repo, token, &config.files.repo_url)?; drop(repo); info!( "Document {:?} removed and changes synced to Github with message: {message:?}", @@ -210,36 +209,32 @@ impl Interface { /// If the repository at the provided path exists, open it and fetch the latest changes from the `master` branch. /// If not, clone into the provided path. #[tracing::instrument] - fn load_repository + std::fmt::Debug>(path: P, state: AppState) -> Result { - let config = Arc::clone(&state.config); + fn load_repository + std::fmt::Debug>(path: P, repo_url: String) -> Result { if let Ok(repo) = Repository::open("./repo") { Self::git_pull(&repo)?; info!("Existing repository detected, fetching latest changes..."); return Ok(repo); } - - let repository_url = &config.files.repo_url; + let output_path = Path::new("./repo"); info!( - "No repo detected, cloning {repository_url:?} into {:?}...", + "No repo detected, cloning {repo_url:?} into {:?}...", output_path.display() ); - let repo = Repository::clone(&repository_url, output_path)?; + let repo = Repository::clone(&repo_url, output_path)?; info!("Successfully cloned repo"); Ok(repo) } /// Completely clone and open a new repository, deleting the old one. #[tracing::instrument(skip_all)] - pub fn reclone(&self, state: AppState) -> Result<()> { + pub fn reclone(&self, repo_url: &str) -> Result<()> { // First clone a repo into `repo__tmp`, open that, swap out // TODO: nuke `repo__tmp` if it exists already let repo_path = Path::new("./repo"); // TODO: Possibly implement this path into new config? let tmp_path = Path::new("./repo__tmp"); // TODO: Same here? - let config = Arc::clone(&state.config); info!("Re-cloning repository, temporary repo will be created at {tmp_path:?}"); - let repository_url = &config.files.repo_url; - let tmp_repo = Repository::clone(&repository_url, tmp_path)?; + let tmp_repo = Repository::clone(repo_url, tmp_path)?; info!("Pointing changes to new temp repository"); let mut lock = self.repo.lock().unwrap(); *lock = tmp_repo; @@ -320,11 +315,9 @@ impl Interface { /// /// `token` is a valid Github auth token. // TODO: stop hardcoding refspec and make it an argument. - fn git_push(repo: &Repository, token: &str, state: AppState) -> Result<()> { - let config = Arc::clone(&state.config); - let repository_url = &config.files.repo_url; + fn git_push(repo: &Repository, token: &str, repo_url: &String) -> Result<()> { let authenticated_url = - repository_url.replace("https://", &format!("https://x-access-token:{token}@")); + repo_url.replace("https://", &format!("https://x-access-token:{token}@")); repo.remote_set_pushurl("origin", Some(&authenticated_url))?; let mut remote = repo.find_remote("origin")?; remote.connect(git2::Direction::Push)?; From 2c60523d1c89bef3574656c7dd50eca2887211fe Mon Sep 17 00:00:00 2001 From: Xeckt Date: Mon, 14 Oct 2024 16:22:36 +0100 Subject: [PATCH 06/24] amend reclone.rs for new config --- backend/src/handlers_prelude/reclone.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/handlers_prelude/reclone.rs b/backend/src/handlers_prelude/reclone.rs index 2a18ae4..85d7ab9 100644 --- a/backend/src/handlers_prelude/reclone.rs +++ b/backend/src/handlers_prelude/reclone.rs @@ -11,7 +11,7 @@ pub async fn post_reclone_handler( headers: HeaderMap, ) -> Result<(), (StatusCode, String)> { require_perms(State(&state), headers, &[Permission::ManageUsers]).await?; - state.git.reclone().map_err(eyre_to_axum_err)?; + state.git.reclone(&state.config.files.repo_url).map_err(eyre_to_axum_err)?; Ok(()) } From e1ea6a6de8fc9a305f30c806fa8f02f29e8dc842 Mon Sep 17 00:00:00 2001 From: Xeckt Date: Mon, 14 Oct 2024 16:22:50 +0100 Subject: [PATCH 07/24] add new packages to cargo --- backend/Cargo.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 5d4faac..76bbe5a 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -10,7 +10,6 @@ keywords = ["cms", "wiki"] categories = ["server-backend"] rust-version = "1.75.0" - [dependencies] axum = { version = "0.7.7", features = ["http2", "macros"] } chrono = "0.4.38" @@ -28,3 +27,5 @@ tokio = { version = "1.40.0", features = ["macros", "rt-multi-thread", "signal", tower-http = { version = "0.6.1", features = ["normalize-path", "fs", "cors", "tracing", "trace"] } tracing = "0.1.40" tracing-subscriber = "0.3.18" +toml = "0.8.19" +once_cell = "1.19.0" From 327d7e3be40dd11ef76257793ecf72de521a0ae8 Mon Sep 17 00:00:00 2001 From: Xeckt Date: Mon, 14 Oct 2024 16:23:51 +0100 Subject: [PATCH 08/24] add new config default.toml --- default.toml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 default.toml diff --git a/default.toml b/default.toml new file mode 100644 index 0000000..b1080d0 --- /dev/null +++ b/default.toml @@ -0,0 +1,19 @@ +[files] +asset_path = "docs/" +docs_path = "assets/" +repo_url = "https://github.com/r-Techsupport/rTS_Wiki.git" + +[discord] +admin_username = "xeckt_" + +[oauth.discord] +client_id = "123" +secret = "_1a2B3c" +url = "https://example.com/oauth2" +token_url = "https://discord.com/api/oauth2/token" + +[oauth.github] +client_id = "Iv23libASQn9266tzjpm" + +[database] +url = "sqlite://../hyde-data/data.db" \ No newline at end of file From 6bfdf0230a125982b1cb9760feaf2aac877c96f0 Mon Sep 17 00:00:00 2001 From: Xeckt Date: Mon, 14 Oct 2024 16:26:51 +0100 Subject: [PATCH 09/24] remove unused imports --- backend/src/gh.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/backend/src/gh.rs b/backend/src/gh.rs index decd048..6212d25 100644 --- a/backend/src/gh.rs +++ b/backend/src/gh.rs @@ -6,14 +6,11 @@ use color_eyre::Result; use jsonwebtoken::{encode, Algorithm, EncodingKey, Header}; use reqwest::Client; use serde::{Deserialize, Serialize}; -use std::env; use std::fs::File; use std::io::Read; -use std::os::linux::raw::stat; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; use tokio::sync::Mutex; -use crate::AppState; use crate::hyde_config::HydeConfig; /// In order to authenticate as a github app or generate an installation access token, you must generate a JSON Web Token (JWT). The JWT must contain predefined *claims*. From f4f2b7caa9206e936acbac97601b2ad1c7ba378d Mon Sep 17 00:00:00 2001 From: Xeckt Date: Mon, 14 Oct 2024 16:27:09 +0100 Subject: [PATCH 10/24] fix stupid category stopping me from pushing --- backend/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 76bbe5a..b87f6fb 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -7,7 +7,7 @@ description = "A backend for the r/TechSupport CMS" repository = "https://github.com/r-Techsupport/hyde" readme = "../README.md" keywords = ["cms", "wiki"] -categories = ["server-backend"] +categories = ["web-programming"] rust-version = "1.75.0" [dependencies] From bb9ebb67a0e10ef8aafaf94e2692a9bd0709aa12 Mon Sep 17 00:00:00 2001 From: Xeckt Date: Mon, 14 Oct 2024 16:42:13 +0100 Subject: [PATCH 11/24] didn't push new hyde_config.rs, remove unused imports --- backend/src/git.rs | 2 +- backend/src/hyde_config.rs | 59 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 backend/src/hyde_config.rs diff --git a/backend/src/git.rs b/backend/src/git.rs index 162fc97..16a084e 100644 --- a/backend/src/git.rs +++ b/backend/src/git.rs @@ -2,7 +2,7 @@ use color_eyre::eyre::{bail, ContextCompat}; use color_eyre::{eyre::Context, Result}; -use git2::{AnnotatedCommit, Config, FetchOptions, Oid, Repository, Signature, Status}; +use git2::{AnnotatedCommit, FetchOptions, Oid, Repository, Signature, Status}; use serde::{Deserialize, Serialize}; use std::fs; use std::io::{Read, Write}; diff --git a/backend/src/hyde_config.rs b/backend/src/hyde_config.rs new file mode 100644 index 0000000..5e11149 --- /dev/null +++ b/backend/src/hyde_config.rs @@ -0,0 +1,59 @@ +use std::fs; +use std::sync::Arc; +use serde::Deserialize; + +#[derive(Deserialize, Debug, Clone)] +pub struct HydeConfig { + pub files: Files, + pub discord: Discord, + pub oauth: OAuth, + pub database: Database, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct Files { + pub asset_path: String, + pub docs_path: String, + pub repo_url: String, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct Discord { + pub admin_username: String, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct OAuth { + pub discord: DiscordOAuth, + pub github: GitHubOAuth, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct DiscordOAuth { + pub client_id: String, + pub secret: String, + pub url: String, + pub token_url: String, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct GitHubOAuth { + pub client_id: String, + // Uncomment this if needed + // pub secret: String, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct Database { + pub url: String, +} + +impl HydeConfig { + pub fn load() -> Arc { + let file = fs::read_to_string("default.toml").expect("Unable to read config"); + let config: Self = toml::from_str(&file).expect("Unable to parse config"); + Arc::new(config) + } + + pub fn check() {} +} From 7f98972fe5991621dcdbf2909725495968cdf03a Mon Sep 17 00:00:00 2001 From: Xeckt Date: Mon, 14 Oct 2024 16:44:36 +0100 Subject: [PATCH 12/24] fix ci faults --- backend/src/git.rs | 2 +- backend/src/hyde_config.rs | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/src/git.rs b/backend/src/git.rs index 16a084e..31f7384 100644 --- a/backend/src/git.rs +++ b/backend/src/git.rs @@ -315,7 +315,7 @@ impl Interface { /// /// `token` is a valid Github auth token. // TODO: stop hardcoding refspec and make it an argument. - fn git_push(repo: &Repository, token: &str, repo_url: &String) -> Result<()> { + fn git_push(repo: &Repository, token: &str, repo_url: &str) -> Result<()> { let authenticated_url = repo_url.replace("https://", &format!("https://x-access-token:{token}@")); repo.remote_set_pushurl("origin", Some(&authenticated_url))?; diff --git a/backend/src/hyde_config.rs b/backend/src/hyde_config.rs index 5e11149..50dea76 100644 --- a/backend/src/hyde_config.rs +++ b/backend/src/hyde_config.rs @@ -54,6 +54,5 @@ impl HydeConfig { let config: Self = toml::from_str(&file).expect("Unable to parse config"); Arc::new(config) } - - pub fn check() {} + } From 4c564a27e383c184bddc250c7c060e7feb54036d Mon Sep 17 00:00:00 2001 From: Xeckt Date: Mon, 14 Oct 2024 17:26:18 +0100 Subject: [PATCH 13/24] add comments to the new default.toml file --- default.toml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/default.toml b/default.toml index b1080d0..327ab37 100644 --- a/default.toml +++ b/default.toml @@ -1,19 +1,31 @@ +# Files is related to any URL or internal files Hyde will use [files] +# The location of the markdown files relative to the root of the repo asset_path = "docs/" +# The location of the assets files relative to the root of the repo docs_path = "assets/" +# The URL of the jekyll repository to interface with repo_url = "https://github.com/r-Techsupport/rTS_Wiki.git" [discord] -admin_username = "xeckt_" +admin_username = "xeckt_" # The discord username of the admin account [oauth.discord] +# Put your Discord OAuth2 client ID here client_id = "123" +# Discord Application Secret will go here. DO NOT Share or commit to any source control. secret = "_1a2B3c" +# URL of the application scope you have generated url = "https://example.com/oauth2" +# The OAuth2 token URL (leave untouched if using discord) token_url = "https://discord.com/api/oauth2/token" [oauth.github] +# Github Application Client ID client_id = "Iv23libASQn9266tzjpm" [database] +# This is used for the sqlx tooling/compile checking, but is not directly used +# in the code right now. This should be set to the path of the database relative +# to the backend folder url = "sqlite://../hyde-data/data.db" \ No newline at end of file From feb1c41aace10d7b17b6abc773d3134c80a1ded7 Mon Sep 17 00:00:00 2001 From: Xeckt Date: Mon, 14 Oct 2024 17:26:34 +0100 Subject: [PATCH 14/24] formatting --- default.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/default.toml b/default.toml index 327ab37..093ad49 100644 --- a/default.toml +++ b/default.toml @@ -8,7 +8,8 @@ docs_path = "assets/" repo_url = "https://github.com/r-Techsupport/rTS_Wiki.git" [discord] -admin_username = "xeckt_" # The discord username of the admin account +# The discord username of the admin account +admin_username = "xeckt_" [oauth.discord] # Put your Discord OAuth2 client ID here From 23fdf7e09b1049639a6e5dc5b2505bb93c808346 Mon Sep 17 00:00:00 2001 From: Xeckt Date: Mon, 14 Oct 2024 17:27:48 +0100 Subject: [PATCH 15/24] update comments for default.toml --- default.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/default.toml b/default.toml index 093ad49..04f815c 100644 --- a/default.toml +++ b/default.toml @@ -7,10 +7,12 @@ docs_path = "assets/" # The URL of the jekyll repository to interface with repo_url = "https://github.com/r-Techsupport/rTS_Wiki.git" +# Discord is related to discord specific information to pass to Hyde. [discord] # The discord username of the admin account admin_username = "xeckt_" +# OAuth for Discord and GitHub, handles passing all relevant information to clients in Hyde [oauth.discord] # Put your Discord OAuth2 client ID here client_id = "123" @@ -25,6 +27,7 @@ token_url = "https://discord.com/api/oauth2/token" # Github Application Client ID client_id = "Iv23libASQn9266tzjpm" +# Database for anything database related Hyde will utilise. [database] # This is used for the sqlx tooling/compile checking, but is not directly used # in the code right now. This should be set to the path of the database relative From 8668ad6024f69467b6b745e152d01bfc7ad88548 Mon Sep 17 00:00:00 2001 From: Xeckt Date: Wed, 16 Oct 2024 10:09:10 +0100 Subject: [PATCH 16/24] move hydeconfig to only handlers and pass necessary data to handler logic --- backend/src/gh.rs | 22 +++++++++++----------- backend/src/git.rs | 8 ++++---- backend/src/handlers_prelude/doc.rs | 4 ++-- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/backend/src/gh.rs b/backend/src/gh.rs index 6212d25..0a0cd10 100644 --- a/backend/src/gh.rs +++ b/backend/src/gh.rs @@ -43,11 +43,11 @@ struct Claims { } impl Claims { - pub fn new(config: Arc) -> Result { + pub fn new(client_id: &str) -> Result { let current_time = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); let iat = current_time - 60; let exp = current_time + (60 * 5); - let iss = config.oauth.github.client_id.clone(); + let iss = client_id.to_string(); Ok(Self { iat, @@ -76,12 +76,12 @@ impl GithubAccessToken { } /// Return the cached token if it's less than one hour old, or fetch a new token from the api, and return that, updating the cache - pub async fn get(&self, req_client: &Client, config: Arc) -> Result { + pub async fn get(&self, req_client: &Client, client_id: &str) -> Result { let mut token_ref = self.token.lock().await; // Fetch a new token if more than 59 minutes have passed // Tokens expire after 1 hour, this is to account for clock drift if SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs() > (60 * 59) { - let api_response = get_access_token(req_client, config).await?; + let api_response = get_access_token(req_client, client_id).await?; *token_ref = api_response.0; let mut expires_ref = self.expires_at.lock().await; *expires_ref = api_response.1; @@ -99,12 +99,12 @@ struct AccessTokenResponse { /// Request a github installation access token using the provided reqwest client. /// The installation access token will expire after 1 hour. /// Returns the new token, and the time of expiration -async fn get_access_token(req_client: &Client, config: Arc) -> Result<(String, SystemTime)> { - let token = gen_jwt_token(Arc::clone(&config))?; +async fn get_access_token(req_client: &Client, client_id: &str) -> Result<(String, SystemTime)> { + let token = gen_jwt_token(client_id)?; let response = req_client .post(format!( "https://api.github.com/app/installations/{}/access_tokens", - get_installation_id(req_client, Arc::clone(&config)).await? + get_installation_id(req_client, client_id).await? )) .bearer_auth(token) .header("Accept", "application/vnd.github+json") @@ -129,10 +129,10 @@ struct InstallationIdResponse { /// Fetch the Installation ID. This value is required for most API calls /// /// -async fn get_installation_id(req_client: &Client, config: Arc) -> Result { +async fn get_installation_id(req_client: &Client, client_id: &str) -> Result { let response = req_client .get("https://api.github.com/app/installations") - .bearer_auth(gen_jwt_token(config)?) + .bearer_auth(gen_jwt_token(client_id)?) .header("User-Agent", "Hyde") // https://docs.github.com/en/rest/about-the-rest-api/api-versions?apiVersion=2022-11-28 .header("X-GitHub-Api-Version", "2022-11-28") @@ -151,14 +151,14 @@ async fn get_installation_id(req_client: &Client, config: Arc) -> Re } /// Generate a new JWT token for use with github api interactions. -fn gen_jwt_token(config: Arc) -> Result { +fn gen_jwt_token(client_id: &str) -> Result { let mut private_key_file = File::open("hyde-data/key.pem") .wrap_err("Failed to read private key from `hyde-data/key.pem`")?; let mut private_key = Vec::new(); private_key_file.read_to_end(&mut private_key)?; Ok(encode( &Header::new(Algorithm::RS256), - &Claims::new(config)?, + &Claims::new(client_id)?, &EncodingKey::from_rsa_pem(&private_key)?, )?) } diff --git a/backend/src/git.rs b/backend/src/git.rs index 31f7384..feb9436 100644 --- a/backend/src/git.rs +++ b/backend/src/git.rs @@ -177,26 +177,26 @@ impl Interface { // it creates errors that note the destructor for other values failing because of it (tree) pub fn delete_doc + Copy>( &self, - config: Arc, + doc_path: &str, + repo_url: &str, path: P, message: &str, token: &str, ) -> Result<()> { - let config = Arc::clone(&config); let repo = self.repo.lock().unwrap(); let mut path_to_doc: PathBuf = PathBuf::new(); path_to_doc.push(&self.doc_path); path_to_doc.push(path); let msg = format!("[Hyde]: {message}"); // Relative to the root of the repo, not the current dir, so typically `./docs` instead of `./repo/docs` - let mut relative_path = PathBuf::from(&config.files.docs_path); + let mut relative_path = PathBuf::from(doc_path); // Standard practice is to stage commits by adding them to an index. relative_path.push(path); fs::remove_file(&path_to_doc).wrap_err_with(|| format!("Failed to remove document the document at {path_to_doc:?}"))?; Self::git_add(&repo, ".")?; let commit_id = Self::git_commit(&repo, msg, None)?; debug!("New commit made with ID: {:?}", commit_id); - Self::git_push(&repo, token, &config.files.repo_url)?; + Self::git_push(&repo, token, repo_url)?; drop(repo); info!( "Document {:?} removed and changes synced to Github with message: {message:?}", diff --git a/backend/src/handlers_prelude/doc.rs b/backend/src/handlers_prelude/doc.rs index c646a22..1babc9f 100644 --- a/backend/src/handlers_prelude/doc.rs +++ b/backend/src/handlers_prelude/doc.rs @@ -66,7 +66,7 @@ pub async fn put_doc_handler( ) .await?; - let gh_token = match &state.gh_credentials.get(&state.reqwest_client, Arc::clone(&state.config)).await { + let gh_token = match &state.gh_credentials.get(&state.reqwest_client, &state.config.oauth.github.client_id).await { Ok(t) => t.clone(), Err(e) => { error!("Failed to authenticate with github for a put_doc request with error: {e:?}"); @@ -109,7 +109,7 @@ pub async fn delete_doc_handler( .await?; let gh_token = state.gh_credentials.get(&state.reqwest_client, Arc::clone(&state.config)).await.unwrap(); - state.git.delete_doc(Arc::clone(&state.config), &query.path, &format!("{} deleted {}", author.username, query.path), &gh_token).map_err(eyre_to_axum_err)?; + state.git.delete_doc(&state.config.files.docs_path, &state.config.files.repo_url, &query.path, &format!("{} deleted {}", author.username, query.path), &gh_token).map_err(eyre_to_axum_err)?; Ok(StatusCode::NO_CONTENT) } From 42a7b00d41e45d7a82f1e4276b104397b64073a2 Mon Sep 17 00:00:00 2001 From: Xeckt Date: Wed, 16 Oct 2024 10:09:55 +0100 Subject: [PATCH 17/24] refactor hydeconfig to app_conf --- backend/src/{hyde_config.rs => app_conf.rs} | 6 +++--- backend/src/main.rs | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) rename backend/src/{hyde_config.rs => app_conf.rs} (96%) diff --git a/backend/src/hyde_config.rs b/backend/src/app_conf.rs similarity index 96% rename from backend/src/hyde_config.rs rename to backend/src/app_conf.rs index 50dea76..f39eb0a 100644 --- a/backend/src/hyde_config.rs +++ b/backend/src/app_conf.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use serde::Deserialize; #[derive(Deserialize, Debug, Clone)] -pub struct HydeConfig { +pub struct AppConf { pub files: Files, pub discord: Discord, pub oauth: OAuth, @@ -48,11 +48,11 @@ pub struct Database { pub url: String, } -impl HydeConfig { +impl AppConf { pub fn load() -> Arc { let file = fs::read_to_string("default.toml").expect("Unable to read config"); let config: Self = toml::from_str(&file).expect("Unable to parse config"); + Arc::new(config) } - } diff --git a/backend/src/main.rs b/backend/src/main.rs index 76ed114..672d089 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -8,7 +8,7 @@ mod gh; pub mod git; mod handlers_prelude; pub mod perms; -mod hyde_config; +mod app_conf; use axum::{ extract::MatchedPath, @@ -46,12 +46,12 @@ use tokio::task; use tower_http::cors::CorsLayer; use tower_http::trace::TraceLayer; use tower_http::{normalize_path::NormalizePathLayer, services::ServeDir}; -use crate::hyde_config::HydeConfig; +use crate::app_conf::AppConf; /// Global app state passed to handlers by axum #[derive(Clone)] pub struct AppState { - pub config: Arc, + pub config: Arc, git: git::Interface, oauth: BasicClient, reqwest_client: Client, @@ -134,7 +134,7 @@ async fn main() -> Result<()> { /// Initialize an instance of [`AppState`] #[tracing::instrument] async fn init_state() -> Result { - let config = HydeConfig::load(); + let config = AppConf::load(); let repo_url = config.files.repo_url.clone(); let git = task::spawn(async { git::Interface::new(repo_url)}).await??; let reqwest_client = Client::new(); From a40bd38722eb81b89bf5940c241bd81b11421b39 Mon Sep 17 00:00:00 2001 From: Xeckt Date: Wed, 16 Oct 2024 10:11:15 +0100 Subject: [PATCH 18/24] update delete doc handler to use client_id for get function --- backend/src/handlers_prelude/doc.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/handlers_prelude/doc.rs b/backend/src/handlers_prelude/doc.rs index 1babc9f..ee1035e 100644 --- a/backend/src/handlers_prelude/doc.rs +++ b/backend/src/handlers_prelude/doc.rs @@ -108,7 +108,7 @@ pub async fn delete_doc_handler( ) .await?; - let gh_token = state.gh_credentials.get(&state.reqwest_client, Arc::clone(&state.config)).await.unwrap(); + let gh_token = state.gh_credentials.get(&state.reqwest_client, &state.config.oauth.github.client_id).await.unwrap(); state.git.delete_doc(&state.config.files.docs_path, &state.config.files.repo_url, &query.path, &format!("{} deleted {}", author.username, query.path), &gh_token).map_err(eyre_to_axum_err)?; Ok(StatusCode::NO_CONTENT) From 9a60da9f3cd5ae67b981d6824ed125c5534d8ca3 Mon Sep 17 00:00:00 2001 From: Xeckt Date: Wed, 16 Oct 2024 10:11:27 +0100 Subject: [PATCH 19/24] remove username from default.toml --- default.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/default.toml b/default.toml index 04f815c..c8ec042 100644 --- a/default.toml +++ b/default.toml @@ -10,7 +10,7 @@ repo_url = "https://github.com/r-Techsupport/rTS_Wiki.git" # Discord is related to discord specific information to pass to Hyde. [discord] # The discord username of the admin account -admin_username = "xeckt_" +admin_username = "username" # OAuth for Discord and GitHub, handles passing all relevant information to clients in Hyde [oauth.discord] From 525704471292f0d66af095c616e96b11a5dac225 Mon Sep 17 00:00:00 2001 From: Xeckt Date: Wed, 16 Oct 2024 10:11:33 +0100 Subject: [PATCH 20/24] delete default.env --- default.env | 34 ---------------------------------- 1 file changed, 34 deletions(-) delete mode 100644 default.env diff --git a/default.env b/default.env deleted file mode 100644 index c50d750..0000000 --- a/default.env +++ /dev/null @@ -1,34 +0,0 @@ -# The discord username of the admin account -ADMIN_USERNAME=your_username - -# The URL of the jekyll repository to interface with -REPO_URL=https://github.com/r-Techsupport/rTS_Wiki.git - -# The location of the markdown files relative to the root of the repo -DOC_PATH=docs/ - -# The location of the assets files relative to the root of the repo -ASSET_PATH=assets/ - -# Put your OAuth2 client ID here -OAUTH_CLIENT_ID=123 - -# Put your OAuth2 client secret here (do *NOT* share or commit to source control) -OAUTH_SECRET=_1a2B3c - -# The OAuth2 base authorization URL generated in the Discord developer dashboard. -# If using discord, this'll look something like https://discord.com/oauth2/authorize?... -OAUTH_URL=https://example.com/oauth2 - -# The OAuth2 token URL (leave untouched if using discord) -# TODO: because we're only support discord at this time, this can be removed, -# and the config file generally made cleaner -OAUTH_TOKEN_URL=https://discord.com/api/oauth2/token - -# Github Application Client ID -GH_CLIENT_ID=Iv23libASQn9266tzjpm - -# This is used for the sqlx tooling/compile checking, but is not directly used -# in the code right now. This should be set to the path of the database relative -# to the backend folder -DATABASE_URL=sqlite://../hyde-data/data.db From d9b88b98b05b6a70ae31a776b86db0ad86e093c0 Mon Sep 17 00:00:00 2001 From: Xeckt Date: Wed, 16 Oct 2024 10:11:46 +0100 Subject: [PATCH 21/24] remove unused dependency --- backend/Cargo.lock | 53 ++++++++++++++++++++++++++++++++++++++++++++++ backend/Cargo.toml | 1 - 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 8cdfb65..6b5fcd0 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -936,6 +936,7 @@ dependencies = [ "serde_json", "sqlx", "tokio", + "toml", "tower-http", "tracing", "tracing-subscriber", @@ -2007,6 +2008,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2602,6 +2612,40 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower" version = "0.4.13" @@ -3170,6 +3214,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.50.0" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index b87f6fb..9ec00bb 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -28,4 +28,3 @@ tower-http = { version = "0.6.1", features = ["normalize-path", "fs", "cors", "t tracing = "0.1.40" tracing-subscriber = "0.3.18" toml = "0.8.19" -once_cell = "1.19.0" From 0fb7ccfe2f374b196ba49a6c725234cd4b2a7847 Mon Sep 17 00:00:00 2001 From: Xeckt Date: Wed, 16 Oct 2024 10:11:58 +0100 Subject: [PATCH 22/24] remove unused import from gh.rs --- backend/src/gh.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/src/gh.rs b/backend/src/gh.rs index 0a0cd10..bcc31b2 100644 --- a/backend/src/gh.rs +++ b/backend/src/gh.rs @@ -11,7 +11,6 @@ use std::io::Read; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; use tokio::sync::Mutex; -use crate::hyde_config::HydeConfig; /// In order to authenticate as a github app or generate an installation access token, you must generate a JSON Web Token (JWT). The JWT must contain predefined *claims*. /// From bdfcf7584227e3723220ac23c0a78304642115d4 Mon Sep 17 00:00:00 2001 From: Xeckt Date: Wed, 16 Oct 2024 10:12:10 +0100 Subject: [PATCH 23/24] rename hydeconfig to appconf in git.rs --- backend/src/git.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/git.rs b/backend/src/git.rs index feb9436..644315c 100644 --- a/backend/src/git.rs +++ b/backend/src/git.rs @@ -12,7 +12,7 @@ use std::{ sync::{Arc, Mutex}, }; use tracing::{debug, info, warn}; -use crate::hyde_config::HydeConfig; +use crate::app_conf::AppConf; #[derive(Clone)] pub struct Interface { @@ -123,7 +123,7 @@ impl Interface { #[tracing::instrument(skip_all)] pub fn put_doc + Copy + std::fmt::Debug>( &self, - config: Arc, + config: Arc, path: P, new_doc: &str, message: &str, From c33f7331fb6db38d4bd8ad3cd24f3ffeabfa0e95 Mon Sep 17 00:00:00 2001 From: Xeckt Date: Wed, 16 Oct 2024 12:00:02 +0100 Subject: [PATCH 24/24] add config validation and to exit on empty configuration values --- backend/src/app_conf.rs | 70 +++++++++++++++++++++++++++++++++++------ backend/src/main.rs | 3 +- 2 files changed, 62 insertions(+), 11 deletions(-) diff --git a/backend/src/app_conf.rs b/backend/src/app_conf.rs index f39eb0a..ba8512f 100644 --- a/backend/src/app_conf.rs +++ b/backend/src/app_conf.rs @@ -1,8 +1,9 @@ -use std::fs; +use std::{fs, process}; use std::sync::Arc; use serde::Deserialize; +use tracing::{info, error}; -#[derive(Deserialize, Debug, Clone)] +#[derive(Deserialize, Debug, Clone, Default, PartialEq, Eq)] pub struct AppConf { pub files: Files, pub discord: Discord, @@ -10,25 +11,25 @@ pub struct AppConf { pub database: Database, } -#[derive(Deserialize, Debug, Clone)] +#[derive(Deserialize, Debug, Clone, Default, PartialEq, Eq)] pub struct Files { pub asset_path: String, pub docs_path: String, pub repo_url: String, } -#[derive(Deserialize, Debug, Clone)] +#[derive(Deserialize, Debug, Clone, Default, PartialEq, Eq)] pub struct Discord { pub admin_username: String, } -#[derive(Deserialize, Debug, Clone)] +#[derive(Deserialize, Debug, Clone, Default, PartialEq, Eq)] pub struct OAuth { pub discord: DiscordOAuth, pub github: GitHubOAuth, } -#[derive(Deserialize, Debug, Clone)] +#[derive(Deserialize, Debug, Clone, Default, PartialEq, Eq)] pub struct DiscordOAuth { pub client_id: String, pub secret: String, @@ -36,23 +37,74 @@ pub struct DiscordOAuth { pub token_url: String, } -#[derive(Deserialize, Debug, Clone)] +#[derive(Deserialize, Debug, Clone, Default, PartialEq, Eq)] pub struct GitHubOAuth { pub client_id: String, // Uncomment this if needed // pub secret: String, } -#[derive(Deserialize, Debug, Clone)] +#[derive(Deserialize, Debug, Clone, Default, PartialEq, Eq)] pub struct Database { pub url: String, } +// Trait to validate fields in each struct +trait ValidateFields { + fn validate(&self, path: &str) -> Result<(), String>; +} + +// Macro to validate all fields for each struct +macro_rules! impl_validate { + ($struct_name:ident, $( $field:ident ),* ) => { + impl ValidateFields for $struct_name { + fn validate(&self, path: &str) -> Result<(), String> { + $( + let field_path = format!("{}.{}", path, stringify!($field)); + if self.$field.is_empty() { + return Err(format!("Field '{}' is empty", field_path)); + } + )* + Ok(()) + } + } + }; +} + +impl_validate!(Files, asset_path, docs_path, repo_url); +impl_validate!(Discord, admin_username); +impl_validate!(DiscordOAuth, client_id, secret, url, token_url); +impl_validate!(GitHubOAuth, client_id); +impl_validate!(Database, url); + +impl ValidateFields for OAuth { + fn validate(&self, path: &str) -> Result<(), String> { + self.discord.validate(&format!("{}.discord", path))?; + self.github.validate(&format!("{}.github", path))?; + Ok(()) + } +} + +impl ValidateFields for AppConf { + fn validate(&self, path: &str) -> Result<(), String> { + self.files.validate(&format!("{}.files", path))?; + self.discord.validate(&format!("{}.discord", path))?; + self.oauth.validate(&format!("{}.oauth", path))?; + self.database.validate(&format!("{}.database", path))?; + Ok(()) + } +} impl AppConf { pub fn load() -> Arc { let file = fs::read_to_string("default.toml").expect("Unable to read config"); let config: Self = toml::from_str(&file).expect("Unable to parse config"); - + match config.validate("config") { + Ok(_) => info!("Configuration isn't empty"), + Err(e) => { + error!("Validation error: {}", e); + process::exit(1) + }, + } Arc::new(config) } } diff --git a/backend/src/main.rs b/backend/src/main.rs index 672d089..5b51213 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -26,8 +26,7 @@ use db::Database; use gh::GithubAccessToken; use handlers_prelude::*; #[cfg(target_family = "unix")] -use tracing::error; -use tracing::{debug, info, info_span, warn}; +use tracing::{debug, info, info_span, warn, error}; // use tracing_subscriber::filter::LevelFilter; use oauth2::{basic::BasicClient, AuthUrl, ClientId, ClientSecret, TokenUrl}; use reqwest::{