diff --git a/Cargo.toml b/Cargo.toml index 063403b9..514998ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -79,6 +79,7 @@ dmfr-folder-reader = {git = "https://github.com/catenarytransit/dmfr-folder-read tokio-zookeeper = "0.2.1" uuid = "1.8.0" zip = "0.6.6" +git2 = "0.18.3" [[bin]] name = "maple" diff --git a/src/maple/main.rs b/src/maple/main.rs index 97e5675d..3c3d35ac 100644 --- a/src/maple/main.rs +++ b/src/maple/main.rs @@ -22,6 +22,7 @@ mod transitland_download; use chateau::chateau; use dmfr_folder_reader::read_folders; use dmfr_folder_reader::ReturnDmfrAnalysis; +use git2::Repository; use crate::transitland_download::DownloadedFeedsInformation; @@ -29,7 +30,37 @@ async fn run_ingest() -> Result<(), Box> { const maple_ingestion_version: i32 = 1; - // TODO! Ensure git submodule transitland-atlas downloads and updates correctly + //Ensure git submodule transitland-atlas downloads and updates correctly + match Repository::open("./") { + Ok(repo) => { + match repo.find_submodule("transitland-atlas") { + Ok(transitland_submodule) => { + println!("Submodule found."); + + let mut transitland_submodule = transitland_submodule; + + match transitland_submodule.update(true, None) { + Ok(update) => { + println!("Submodule updated."); + }, + Err(update_err) => { + eprintln!("Unable to update submodule"); + + // don't need to fail if can't reach github servers for now + } + } + }, + Err(find_submodule) => { + eprintln!("Can't find submodule!"); + return Err(Box::new(find_submodule)); + } + } + }, + Err(repo_err) => { + eprintln!("Can't find own repo!"); + return Err(Box::new(repo_err)); + } + } //These feeds should be discarded because they are duplicated in a larger dataset called `f-sf~bay~area~rg`, which has everything in a single zip file let feeds_to_discard: HashSet<&str> = HashSet::from_iter(vec![ diff --git a/src/maple/transitland_download.rs b/src/maple/transitland_download.rs index fec0d051..f037a42d 100644 --- a/src/maple/transitland_download.rs +++ b/src/maple/transitland_download.rs @@ -60,7 +60,7 @@ pub async fn download_return_eligible_feeds( transitland_meta: &ReturnDmfrAnalysis, pool: &sqlx::Pool, ) -> Result, ()> { - let threads: usize = 16; + let threads: usize = 32; let _ = fs::create_dir("gtfs_static_zips");