From a224aaf77d7d749254741b899fa10a06e762569c Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Fri, 1 Sep 2023 18:58:01 +0200 Subject: [PATCH 01/51] feat(cli): scaffold `seq sort` cli command, input parsing and main loop --- packages_rs/nextclade-cli/src/cli/mod.rs | 1 + .../nextclade-cli/src/cli/nextclade_cli.rs | 64 +++++++++- .../src/cli/nextclade_seq_sort.rs | 110 ++++++++++++++++++ packages_rs/nextclade/src/io/dataset.rs | 11 ++ packages_rs/nextclade/src/lib.rs | 1 + .../nextclade/src/sort/minimizer_index.rs | 105 +++++++++++++++++ .../nextclade/src/sort/minimizer_search.rs | 20 ++++ packages_rs/nextclade/src/sort/mod.rs | 2 + 8 files changed, 313 insertions(+), 1 deletion(-) create mode 100644 packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs create mode 100644 packages_rs/nextclade/src/sort/minimizer_index.rs create mode 100644 packages_rs/nextclade/src/sort/minimizer_search.rs create mode 100644 packages_rs/nextclade/src/sort/mod.rs diff --git a/packages_rs/nextclade-cli/src/cli/mod.rs b/packages_rs/nextclade-cli/src/cli/mod.rs index 9647610b4..1a058aba2 100644 --- a/packages_rs/nextclade-cli/src/cli/mod.rs +++ b/packages_rs/nextclade-cli/src/cli/mod.rs @@ -3,4 +3,5 @@ pub mod nextclade_dataset_get; pub mod nextclade_dataset_list; pub mod nextclade_loop; pub mod nextclade_ordered_writer; +pub mod nextclade_seq_sort; pub mod verbosity; diff --git a/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs b/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs index b40772240..a35f34c0e 100644 --- a/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs +++ b/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs @@ -1,6 +1,7 @@ use crate::cli::nextclade_dataset_get::nextclade_dataset_get; use crate::cli::nextclade_dataset_list::nextclade_dataset_list; use crate::cli::nextclade_loop::nextclade_run; +use crate::cli::nextclade_seq_sort::nextclade_seq_sort; use crate::cli::verbosity::{Verbosity, WarnLevel}; use crate::io::http_client::ProxyConfig; use clap::{ArgGroup, CommandFactory, Parser, Subcommand, ValueEnum, ValueHint}; @@ -80,8 +81,13 @@ pub enum NextcladeCommands { /// List and download available Nextclade datasets /// - /// For short help type: `nextclade -h`, for extended help type: `nextclade --help`. Each subcommand has its own help, for example: `nextclade run --help`. + /// For short help type: `nextclade -h`, for extended help type: `nextclade --help`. Each subcommand has its own help, for example: `nextclade dataset --help`. Dataset(Box), + + /// Perform operations on sequences + /// + /// For short help type: `nextclade -h`, for extended help type: `nextclade --help`. Each subcommand has its own help, for example: `nextclade seq --help`. + Seq(Box), } #[derive(Parser, Debug)] @@ -618,6 +624,59 @@ pub struct NextcladeRunArgs { pub other_params: NextcladeRunOtherParams, } +#[derive(Parser, Debug)] +pub struct NextcladeSeqArgs { + #[clap(subcommand)] + pub command: NextcladeSeqCommands, +} + +#[derive(Subcommand, Debug)] +#[clap(verbatim_doc_comment)] +pub enum NextcladeSeqCommands { + /// Group (sort) input sequences according to the inferred dataset (pathogen) + /// + /// For short help type: `nextclade -h`, for extended help type: `nextclade --help`. Each subcommand has its own help, for example: `nextclade seq sort --help`. + Sort(NextcladeSeqSortArgs), +} + +#[allow(clippy::struct_excessive_bools)] +#[derive(Parser, Debug)] +#[clap(verbatim_doc_comment)] +pub struct NextcladeSeqSortArgs { + /// Path to one or multiple FASTA files with input sequences + /// + /// Supports the following compression formats: "gz", "bz2", "xz", "zst". If no files provided, the plain fasta input is read from standard input (stdin). + /// + /// See: https://en.wikipedia.org/wiki/FASTA_format + #[clap(value_hint = ValueHint::FilePath)] + #[clap(display_order = 1)] + pub input_fastas: Vec, + + /// Path to output directory + /// + /// Sequences will be written in subdirectories: one subdirectory per dataset. Sequences inferred to be belonging to a particular dataset wil lbe places in the corresponding subdirectory. The subdirectory tree can be nested, depending on how dataset names are organized. + /// + #[clap(long)] + #[clap(value_hint = ValueHint::DirPath)] + #[clap(hide_long_help = true, hide_short_help = true)] + pub output_dir: Option, + + /// Use custom dataset server. + /// + /// You can host your own dataset server, with one or more datasets, grouped into dataset collections, and use this server to provide datasets to users of Nextclade CLI and Nextclade Web. Refer to Nextclade dataset documentation for more details. + #[clap(long)] + #[clap(value_hint = ValueHint::Url)] + #[clap(default_value_t = Url::from_str(DATA_FULL_DOMAIN).expect("Invalid URL"))] + pub server: Url, + + #[clap(flatten)] + pub proxy_config: ProxyConfig, + + /// Number of processing jobs. If not specified, all available CPU threads will be used. + #[clap(global = false, long, short = 'j', default_value_t = num_cpus::get())] + pub jobs: usize, +} + fn generate_completions(shell: &str) -> Result<(), Report> { let mut command = NextcladeArgs::command(); @@ -904,5 +963,8 @@ pub fn nextclade_parse_cli_args() -> Result<(), Report> { NextcladeDatasetCommands::List(dataset_list_args) => nextclade_dataset_list(dataset_list_args), NextcladeDatasetCommands::Get(dataset_get_args) => nextclade_dataset_get(&dataset_get_args), }, + NextcladeCommands::Seq(seq_command) => match seq_command.command { + NextcladeSeqCommands::Sort(seq_sort_args) => nextclade_seq_sort(&seq_sort_args), + }, } } diff --git a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs new file mode 100644 index 000000000..ebe2d3767 --- /dev/null +++ b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs @@ -0,0 +1,110 @@ +use crate::cli::nextclade_cli::NextcladeSeqSortArgs; +use crate::dataset::dataset_download::download_datasets_index_json; +use crate::io::http_client::HttpClient; +use eyre::{Report, WrapErr}; +use log::{info, LevelFilter}; +use nextclade::io::fasta::{FastaReader, FastaRecord}; +use nextclade::make_error; +use nextclade::sort::minimizer_index::{MinimizerIndexJson, MINIMIZER_INDEX_ALGO_VERSION}; +use nextclade::sort::minimizer_search::{run_minimizer_search, MinimizerSearchResult}; + +#[derive(Debug, Clone)] +struct MinimizerSearchRecord { + pub fasta_record: FastaRecord, + pub result: MinimizerSearchResult, +} + +pub fn nextclade_seq_sort(args: &NextcladeSeqSortArgs) -> Result<(), Report> { + let NextcladeSeqSortArgs { + server, proxy_config, .. + } = args; + + let verbose = log::max_level() > LevelFilter::Info; + + let mut http = HttpClient::new(server, proxy_config, verbose)?; + + let index = download_datasets_index_json(&mut http)?; + + let minimizer_index_path = index + .minimizer_index + .iter() + .find(|minimizer_index| MINIMIZER_INDEX_ALGO_VERSION == minimizer_index.version) + .map(|minimizer_index| &minimizer_index.path); + + if let Some(minimizer_index_path) = minimizer_index_path { + let minimizer_index_str = http.get(minimizer_index_path)?; + let minimizer_index = MinimizerIndexJson::from_str(String::from_utf8(minimizer_index_str)?)?; + run(args, &minimizer_index) + } else { + make_error!("No supported reference search index data is found for this dataset sever. Try to to upgrade Nextclade to the latest version or contain dataset server maintainers.") + } +} + +pub fn run(args: &NextcladeSeqSortArgs, minimizer_index: &MinimizerIndexJson) -> Result<(), Report> { + let NextcladeSeqSortArgs { + input_fastas, + output_dir, + server, + proxy_config, + jobs, + } = args; + + std::thread::scope(|s| { + const CHANNEL_SIZE: usize = 128; + let (fasta_sender, fasta_receiver) = crossbeam_channel::bounded::(CHANNEL_SIZE); + let (result_sender, result_receiver) = crossbeam_channel::bounded::(CHANNEL_SIZE); + + s.spawn(|| { + let mut reader = FastaReader::from_paths(input_fastas).unwrap(); + loop { + let mut record = FastaRecord::default(); + reader.read(&mut record).unwrap(); + if record.is_empty() { + break; + } + fasta_sender + .send(record) + .wrap_err("When sending a FastaRecord") + .unwrap(); + } + drop(fasta_sender); + }); + + for _ in 0..*jobs { + let fasta_receiver = fasta_receiver.clone(); + let result_sender = result_sender.clone(); + + s.spawn(move || { + let result_sender = result_sender.clone(); + + for fasta_record in &fasta_receiver { + info!("Processing sequence '{}'", fasta_record.seq_name); + + let result = run_minimizer_search(&fasta_record) + .wrap_err_with(|| { + format!( + "When processing sequence #{} '{}'", + fasta_record.index, fasta_record.seq_name + ) + }) + .unwrap(); + + result_sender + .send(MinimizerSearchRecord { fasta_record, result }) + .wrap_err("When sending minimizer record into the channel") + .unwrap(); + } + + drop(result_sender); + }); + } + + let writer = s.spawn(move || { + for record in result_receiver { + println!("{}", &record.fasta_record.seq_name); + } + }); + }); + + Ok(()) +} diff --git a/packages_rs/nextclade/src/io/dataset.rs b/packages_rs/nextclade/src/io/dataset.rs index 653af23dd..402437a20 100644 --- a/packages_rs/nextclade/src/io/dataset.rs +++ b/packages_rs/nextclade/src/io/dataset.rs @@ -21,6 +21,8 @@ pub struct DatasetsIndexJson { pub schema_version: String, + pub minimizer_index: Vec, + #[serde(flatten)] pub other: serde_json::Value, } @@ -325,3 +327,12 @@ pub struct DatasetCollectionUrl { #[serde(flatten)] pub other: serde_json::Value, } + +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct MinimizerIndexVersion { + pub version: String, + pub path: String, + #[serde(flatten)] + pub other: serde_json::Value, +} diff --git a/packages_rs/nextclade/src/lib.rs b/packages_rs/nextclade/src/lib.rs index 4ad4d2a09..479947cb6 100644 --- a/packages_rs/nextclade/src/lib.rs +++ b/packages_rs/nextclade/src/lib.rs @@ -9,6 +9,7 @@ pub mod graph; pub mod io; pub mod qc; pub mod run; +pub mod sort; pub mod translate; pub mod tree; pub mod types; diff --git a/packages_rs/nextclade/src/sort/minimizer_index.rs b/packages_rs/nextclade/src/sort/minimizer_index.rs new file mode 100644 index 000000000..f6eda7db4 --- /dev/null +++ b/packages_rs/nextclade/src/sort/minimizer_index.rs @@ -0,0 +1,105 @@ +use crate::io::json::json_parse; +use crate::io::schema_version::{SchemaVersion, SchemaVersionParams}; +use eyre::Report; +use schemars::JsonSchema; +use serde::ser::SerializeMap; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use std::collections::BTreeMap; +use std::str::FromStr; + +pub const MINIMIZER_INDEX_SCHEMA_VERSION_FROM: &str = "3.0.0"; +pub const MINIMIZER_INDEX_SCHEMA_VERSION_TO: &str = "3.0.0"; +pub const MINIMIZER_INDEX_ALGO_VERSION: &str = "1"; + +/// Contains external configuration and data specific for a particular pathogen +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct MinimizerIndexJson { + #[serde(rename = "schemaVersion")] + pub schema_version: String, + + pub version: String, + + pub params: MinimizerParams, + + #[schemars(with = "BTreeMap")] + #[serde(serialize_with = "serde_serialize_minimizers")] + #[serde(deserialize_with = "serde_deserialize_minimizers")] + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub minimizers: BTreeMap, + + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub references: Vec, + + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub normalization: Vec, + + #[serde(flatten)] + pub other: serde_json::Value, +} + +/// Serde serializer for Letter sequences +pub fn serde_serialize_minimizers( + minimizers: &BTreeMap, + s: S, +) -> Result { + let mut map = s.serialize_map(Some(minimizers.len()))?; + for (k, v) in minimizers { + map.serialize_entry(&k.to_string(), &v.to_string())?; + } + map.end() +} + +/// Serde deserializer for Letter sequences +pub fn serde_deserialize_minimizers<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + let map = BTreeMap::::deserialize(deserializer)?; + + let res = map + .into_iter() + .map(|(k, v)| Ok((usize::from_str(&k)?, v))) + .collect::, Report>>() + .unwrap(); + + Ok(res) +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct MinimizerParams { + pub k: i64, + + pub cutoff: i64, + + #[serde(flatten)] + pub other: serde_json::Value, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct MinimizerIndexRefInfo { + pub length: i64, + pub name: String, + pub n_minimizers: i64, + + #[serde(flatten)] + pub other: serde_json::Value, +} + +impl MinimizerIndexJson { + pub fn from_str(s: impl AsRef) -> Result { + let s = s.as_ref(); + + SchemaVersion::check_warn( + s, + &SchemaVersionParams { + name: "minimizer_index.json", + ver_from: Some(MINIMIZER_INDEX_SCHEMA_VERSION_FROM), + ver_to: Some(MINIMIZER_INDEX_SCHEMA_VERSION_TO), + }, + ); + + json_parse(s) + } +} diff --git a/packages_rs/nextclade/src/sort/minimizer_search.rs b/packages_rs/nextclade/src/sort/minimizer_search.rs new file mode 100644 index 000000000..7a4d35f64 --- /dev/null +++ b/packages_rs/nextclade/src/sort/minimizer_search.rs @@ -0,0 +1,20 @@ +use crate::io::fasta::FastaRecord; +use eyre::Report; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct MinimizerSearchResult { + #[serde(rename = "schemaVersion")] + pub schema_version: String, + + pub version: String, +} + +pub fn run_minimizer_search(fasta_record: &FastaRecord) -> Result { + Ok(MinimizerSearchResult { + schema_version: "".to_owned(), + version: "".to_owned(), + }) +} diff --git a/packages_rs/nextclade/src/sort/mod.rs b/packages_rs/nextclade/src/sort/mod.rs new file mode 100644 index 000000000..8c3a0809b --- /dev/null +++ b/packages_rs/nextclade/src/sort/mod.rs @@ -0,0 +1,2 @@ +pub mod minimizer_index; +pub mod minimizer_search; From 16cf1819988e7b26eab5d5a94b175c700bbc8f5b Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Fri, 1 Sep 2023 20:55:25 +0200 Subject: [PATCH 02/51] feat(cli): implement ref search --- .../src/cli/nextclade_seq_sort.rs | 12 +- .../nextclade/src/sort/minimizer_index.rs | 17 +-- .../nextclade/src/sort/minimizer_search.rs | 130 ++++++++++++++++-- 3 files changed, 139 insertions(+), 20 deletions(-) diff --git a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs index ebe2d3767..7653ff62f 100644 --- a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs +++ b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs @@ -7,6 +7,7 @@ use nextclade::io::fasta::{FastaReader, FastaRecord}; use nextclade::make_error; use nextclade::sort::minimizer_index::{MinimizerIndexJson, MINIMIZER_INDEX_ALGO_VERSION}; use nextclade::sort::minimizer_search::{run_minimizer_search, MinimizerSearchResult}; +use nextclade::utils::string::truncate; #[derive(Debug, Clone)] struct MinimizerSearchRecord { @@ -80,7 +81,7 @@ pub fn run(args: &NextcladeSeqSortArgs, minimizer_index: &MinimizerIndexJson) -> for fasta_record in &fasta_receiver { info!("Processing sequence '{}'", fasta_record.seq_name); - let result = run_minimizer_search(&fasta_record) + let result = run_minimizer_search(&fasta_record, minimizer_index) .wrap_err_with(|| { format!( "When processing sequence #{} '{}'", @@ -100,8 +101,15 @@ pub fn run(args: &NextcladeSeqSortArgs, minimizer_index: &MinimizerIndexJson) -> } let writer = s.spawn(move || { + println!("{:40} | {:10} | {:10}", "Seq. name", "total hits", "max hit"); for record in result_receiver { - println!("{}", &record.fasta_record.seq_name); + println!( + "{:40} | {:40} | {:>10} | {:>.3}", + &truncate(record.fasta_record.seq_name, 40), + &truncate(record.result.dataset.unwrap_or_default(), 40), + &record.result.total_hits, + &record.result.max_normalized_hit + ); } }); }); diff --git a/packages_rs/nextclade/src/sort/minimizer_index.rs b/packages_rs/nextclade/src/sort/minimizer_index.rs index f6eda7db4..2d6a7a901 100644 --- a/packages_rs/nextclade/src/sort/minimizer_index.rs +++ b/packages_rs/nextclade/src/sort/minimizer_index.rs @@ -11,6 +11,8 @@ pub const MINIMIZER_INDEX_SCHEMA_VERSION_FROM: &str = "3.0.0"; pub const MINIMIZER_INDEX_SCHEMA_VERSION_TO: &str = "3.0.0"; pub const MINIMIZER_INDEX_ALGO_VERSION: &str = "1"; +pub type MinimizerMap = BTreeMap; + /// Contains external configuration and data specific for a particular pathogen #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] @@ -26,7 +28,7 @@ pub struct MinimizerIndexJson { #[serde(serialize_with = "serde_serialize_minimizers")] #[serde(deserialize_with = "serde_deserialize_minimizers")] #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub minimizers: BTreeMap, + pub minimizers: MinimizerMap, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub references: Vec, @@ -39,10 +41,7 @@ pub struct MinimizerIndexJson { } /// Serde serializer for Letter sequences -pub fn serde_serialize_minimizers( - minimizers: &BTreeMap, - s: S, -) -> Result { +pub fn serde_serialize_minimizers(minimizers: &MinimizerMap, s: S) -> Result { let mut map = s.serialize_map(Some(minimizers.len()))?; for (k, v) in minimizers { map.serialize_entry(&k.to_string(), &v.to_string())?; @@ -51,15 +50,13 @@ pub fn serde_serialize_minimizers( } /// Serde deserializer for Letter sequences -pub fn serde_deserialize_minimizers<'de, D: Deserializer<'de>>( - deserializer: D, -) -> Result, D::Error> { +pub fn serde_deserialize_minimizers<'de, D: Deserializer<'de>>(deserializer: D) -> Result { let map = BTreeMap::::deserialize(deserializer)?; let res = map .into_iter() - .map(|(k, v)| Ok((usize::from_str(&k)?, v))) - .collect::, Report>>() + .map(|(k, v)| Ok((u64::from_str(&k)?, v))) + .collect::>() .unwrap(); Ok(res) diff --git a/packages_rs/nextclade/src/sort/minimizer_search.rs b/packages_rs/nextclade/src/sort/minimizer_search.rs index 7a4d35f64..ce13e45a1 100644 --- a/packages_rs/nextclade/src/sort/minimizer_search.rs +++ b/packages_rs/nextclade/src/sort/minimizer_search.rs @@ -1,20 +1,134 @@ use crate::io::fasta::FastaRecord; +use crate::sort::minimizer_index::{MinimizerIndexJson, MinimizerParams}; use eyre::Report; +use itertools::Itertools; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use std::str::FromStr; #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct MinimizerSearchResult { - #[serde(rename = "schemaVersion")] - pub schema_version: String, + pub dataset: Option, + pub hit_counts: Vec, + pub total_hits: u64, + pub normalized_hits: Vec, + pub max_normalized_hit: f64, +} + +#[allow(clippy::string_slice)] +pub fn run_minimizer_search( + fasta_record: &FastaRecord, + index: &MinimizerIndexJson, +) -> Result { + let normalization = &index.normalization; + let n_refs = index.references.len(); + + let minimizers = get_ref_search_minimizers(fasta_record, &index.params); + let mut hit_counts = vec![0; n_refs]; + for m in minimizers { + if let Some(mz) = index.minimizers.get(&m) { + for i in 0..n_refs { + hit_counts[i] += u64::from_str(&mz[i..=i])?; + } + } + } + + // we expect hits to be proportional to the length of the sequence and the number of minimizers per reference + let mut normalized_hits: Vec = vec![0.0; hit_counts.len()]; + for i in 0..n_refs { + normalized_hits[i] = hit_counts[i] as f64 * normalization[i] / fasta_record.seq.len() as f64; + } + + // require at least 30% of the maximal hits and at least 10 hits + let max_normalized_hit = normalized_hits.iter().copied().fold(0.0, f64::max); + let total_hits: u64 = hit_counts.iter().sum(); + if max_normalized_hit < 0.3 || total_hits < 10 { + Ok(MinimizerSearchResult { + dataset: None, + hit_counts, + total_hits, + normalized_hits, + max_normalized_hit, + }) + } else { + let i_ref = normalized_hits + .iter() + .position_max_by(|x, y| x.total_cmp(y)) + .expect("The `normalized_hits` cannot be empty."); + let reference = &index.references[i_ref]; + Ok(MinimizerSearchResult { + dataset: Some(reference.name.clone()), + hit_counts, + total_hits, + normalized_hits, + max_normalized_hit, + }) + } +} + +const fn invertible_hash(x: u64) -> u64 { + let m: u64 = (1 << 32) - 1; + let mut x: u64 = (!x).wrapping_add(x << 21) & m; + x = x ^ (x >> 24); + x = (x + (x << 3) + (x << 8)) & m; + x = x ^ (x >> 14); + x = (x + (x << 2) + (x << 4)) & m; + x = x ^ (x >> 28); + x = (x + (x << 31)) & m; + x +} + +fn get_hash(kmer: &[u8], params: &MinimizerParams) -> u64 { + let cutoff = params.cutoff as u64; + + let mut x = 0; + let mut j = 0; + + for (i, nuc) in kmer.iter().enumerate() { + let nuc = *nuc as char; + + if i % 3 == 2 { + continue; // skip every third nucleotide to pick up conserved patterns + } + + if !"ACGT".contains(nuc) { + return cutoff + 1; // break out of loop, return hash above cutoff + } + + // A=11=3, C=10=2, G=00=0, T=01=1 + if "AC".contains(nuc) { + x += 1 << j; + } + + if "AT".contains(nuc) { + x += 1 << (j + 1); + } + + j += 2; + } + + invertible_hash(x) +} + +pub fn get_ref_search_minimizers(seq: &FastaRecord, params: &MinimizerParams) -> Vec { + let k = params.k as usize; + let cutoff = params.cutoff as u64; - pub version: String, + let seq_str = preprocess_seq(&seq.seq); + let n = seq_str.len().saturating_sub(k); + let mut minimizers = Vec::with_capacity(n); + for i in 0..n { + let kmer = &seq_str.as_bytes()[i..i + k]; + let mhash = get_hash(kmer, params); + // accept only hashes below cutoff --> reduces the size of the index and the number of lookups + if mhash < cutoff { + minimizers.push(mhash); + } + } + minimizers.into_iter().unique().collect_vec() } -pub fn run_minimizer_search(fasta_record: &FastaRecord) -> Result { - Ok(MinimizerSearchResult { - schema_version: "".to_owned(), - version: "".to_owned(), - }) +fn preprocess_seq(seq: impl AsRef) -> String { + seq.as_ref().to_uppercase().replace('-', "") } From 0cf48f99b551ae86683b8d1ba375e31bdcdea340 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Fri, 1 Sep 2023 21:27:21 +0200 Subject: [PATCH 03/51] feat(cli): allow external minimizer index JSON file --- .../nextclade-cli/src/cli/nextclade_cli.rs | 10 ++++ .../src/cli/nextclade_seq_sort.rs | 50 +++++++++++-------- .../nextclade/src/sort/minimizer_index.rs | 27 +++++++++- 3 files changed, 66 insertions(+), 21 deletions(-) diff --git a/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs b/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs index a35f34c0e..d8007488e 100644 --- a/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs +++ b/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs @@ -652,6 +652,16 @@ pub struct NextcladeSeqSortArgs { #[clap(display_order = 1)] pub input_fastas: Vec, + /// Path to input minimizer index JSON file. + /// + /// By default the latest reference minimizer index is fetched from the dataset server (default or customized with `--server` argument). If this argument is provided, the algorithm skips fetching the default index and uses the index provided in the the JSON file. + /// + /// Supports the following compression formats: "gz", "bz2", "xz", "zst". Use "-" to read uncompressed data from standard input (stdin). + #[clap(long, short = 'm')] + #[clap(value_hint = ValueHint::FilePath)] + #[clap(display_order = 1)] + pub input_minimizer_index_json: Option, + /// Path to output directory /// /// Sequences will be written in subdirectories: one subdirectory per dataset. Sequences inferred to be belonging to a particular dataset wil lbe places in the corresponding subdirectory. The subdirectory tree can be nested, depending on how dataset names are organized. diff --git a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs index 7653ff62f..1c630974f 100644 --- a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs +++ b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs @@ -17,37 +17,44 @@ struct MinimizerSearchRecord { pub fn nextclade_seq_sort(args: &NextcladeSeqSortArgs) -> Result<(), Report> { let NextcladeSeqSortArgs { - server, proxy_config, .. + server, + proxy_config, + input_minimizer_index_json, + .. } = args; let verbose = log::max_level() > LevelFilter::Info; - let mut http = HttpClient::new(server, proxy_config, verbose)?; - - let index = download_datasets_index_json(&mut http)?; - - let minimizer_index_path = index - .minimizer_index - .iter() - .find(|minimizer_index| MINIMIZER_INDEX_ALGO_VERSION == minimizer_index.version) - .map(|minimizer_index| &minimizer_index.path); - - if let Some(minimizer_index_path) = minimizer_index_path { - let minimizer_index_str = http.get(minimizer_index_path)?; - let minimizer_index = MinimizerIndexJson::from_str(String::from_utf8(minimizer_index_str)?)?; - run(args, &minimizer_index) + let minimizer_index = if let Some(input_minimizer_index_json) = &input_minimizer_index_json { + // If a file is provided, use data from it + MinimizerIndexJson::from_path(input_minimizer_index_json) } else { - make_error!("No supported reference search index data is found for this dataset sever. Try to to upgrade Nextclade to the latest version or contain dataset server maintainers.") - } + // Otherwise fetch from dataset server + let mut http = HttpClient::new(server, proxy_config, verbose)?; + let index = download_datasets_index_json(&mut http)?; + let minimizer_index_path = index + .minimizer_index + .iter() + .find(|minimizer_index| MINIMIZER_INDEX_ALGO_VERSION == minimizer_index.version) + .map(|minimizer_index| &minimizer_index.path); + + if let Some(minimizer_index_path) = minimizer_index_path { + let minimizer_index_str = http.get(minimizer_index_path)?; + MinimizerIndexJson::from_str(String::from_utf8(minimizer_index_str)?) + } else { + make_error!("No compatible reference minimizer index data is found for this dataset sever. Cannot proceed. Try to to upgrade Nextclade to the latest version and/or contact dataset server maintainers.") + } + }?; + + run(args, &minimizer_index) } pub fn run(args: &NextcladeSeqSortArgs, minimizer_index: &MinimizerIndexJson) -> Result<(), Report> { let NextcladeSeqSortArgs { input_fastas, output_dir, - server, - proxy_config, jobs, + .. } = args; std::thread::scope(|s| { @@ -101,7 +108,10 @@ pub fn run(args: &NextcladeSeqSortArgs, minimizer_index: &MinimizerIndexJson) -> } let writer = s.spawn(move || { - println!("{:40} | {:10} | {:10}", "Seq. name", "total hits", "max hit"); + println!( + "{:40} | {:40} | {:10} | {:10}", + "Seq. name", "dataset", "total hits", "max hit" + ); for record in result_receiver { println!( "{:40} | {:40} | {:>10} | {:>.3}", diff --git a/packages_rs/nextclade/src/sort/minimizer_index.rs b/packages_rs/nextclade/src/sort/minimizer_index.rs index 2d6a7a901..f7fa222d8 100644 --- a/packages_rs/nextclade/src/sort/minimizer_index.rs +++ b/packages_rs/nextclade/src/sort/minimizer_index.rs @@ -1,10 +1,13 @@ +use crate::io::fs::read_file_to_string; use crate::io::json::json_parse; use crate::io::schema_version::{SchemaVersion, SchemaVersionParams}; -use eyre::Report; +use eyre::{Report, WrapErr}; +use log::warn; use schemars::JsonSchema; use serde::ser::SerializeMap; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use std::collections::BTreeMap; +use std::path::Path; use std::str::FromStr; pub const MINIMIZER_INDEX_SCHEMA_VERSION_FROM: &str = "3.0.0"; @@ -84,7 +87,24 @@ pub struct MinimizerIndexRefInfo { pub other: serde_json::Value, } +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct VersionCheck { + #[serde(rename = "schemaVersion")] + pub version: String, + + #[serde(flatten)] + pub other: serde_json::Value, +} + impl MinimizerIndexJson { + pub fn from_path(filepath: impl AsRef) -> Result { + let filepath = filepath.as_ref(); + let data = + read_file_to_string(filepath).wrap_err_with(|| format!("When reading minimizer index file: {filepath:#?}"))?; + Self::from_str(data) + } + pub fn from_str(s: impl AsRef) -> Result { let s = s.as_ref(); @@ -97,6 +117,11 @@ impl MinimizerIndexJson { }, ); + let VersionCheck { version, .. } = json_parse(s)?; + if version.as_str() > MINIMIZER_INDEX_ALGO_VERSION { + warn!("Version of the minimizer index data ({version}) is greater than maximum supported by this version of Nextclade ({MINIMIZER_INDEX_ALGO_VERSION}). This may lead to errors or incorrect results. Please try to update your version of Nextclade and/or contact dataset maintainers for more details."); + } + json_parse(s) } } From dfa63a60030e12fd3ff40db827513fa422512b5e Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Fri, 1 Sep 2023 21:33:56 +0200 Subject: [PATCH 04/51] feat(cli): improve error message text --- .../nextclade-cli/src/cli/nextclade_seq_sort.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs index 1c630974f..bb78f8f5b 100644 --- a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs +++ b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs @@ -2,6 +2,7 @@ use crate::cli::nextclade_cli::NextcladeSeqSortArgs; use crate::dataset::dataset_download::download_datasets_index_json; use crate::io::http_client::HttpClient; use eyre::{Report, WrapErr}; +use itertools::Itertools; use log::{info, LevelFilter}; use nextclade::io::fasta::{FastaReader, FastaRecord}; use nextclade::make_error; @@ -42,7 +43,18 @@ pub fn nextclade_seq_sort(args: &NextcladeSeqSortArgs) -> Result<(), Report> { let minimizer_index_str = http.get(minimizer_index_path)?; MinimizerIndexJson::from_str(String::from_utf8(minimizer_index_str)?) } else { - make_error!("No compatible reference minimizer index data is found for this dataset sever. Cannot proceed. Try to to upgrade Nextclade to the latest version and/or contact dataset server maintainers.") + let server_versions = index + .minimizer_index + .iter() + .map(|minimizer_index| format!("'{}'", minimizer_index.version)) + .join(","); + let server_versions = if server_versions.is_empty() { + "none".to_owned() + } else { + format!(": {server_versions}") + }; + + make_error!("No compatible reference minimizer index data is found for this dataset sever. Cannot proceed. \n\nThis version of Nextclade supports index versions up to '{}', but the server has{}.\n\nTry to to upgrade Nextclade to the latest version and/or contact dataset server maintainers.", MINIMIZER_INDEX_ALGO_VERSION, server_versions) } }?; From d327c77e471f2b899b560602a490ced52ca3171d Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Fri, 1 Sep 2023 21:41:03 +0200 Subject: [PATCH 05/51] fix(cli): version warning false positive --- packages_rs/nextclade/src/sort/minimizer_index.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/packages_rs/nextclade/src/sort/minimizer_index.rs b/packages_rs/nextclade/src/sort/minimizer_index.rs index f7fa222d8..144ff6262 100644 --- a/packages_rs/nextclade/src/sort/minimizer_index.rs +++ b/packages_rs/nextclade/src/sort/minimizer_index.rs @@ -90,7 +90,6 @@ pub struct MinimizerIndexRefInfo { #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct VersionCheck { - #[serde(rename = "schemaVersion")] pub version: String, #[serde(flatten)] From 8940bbb2bd7c57039ea1c81979e3aa3152531d61 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Fri, 1 Sep 2023 22:07:31 +0200 Subject: [PATCH 06/51] feat(cli): add cli params for minimizer search algo --- .../nextclade-cli/src/cli/nextclade_cli.rs | 11 +++++--- .../src/cli/nextclade_seq_sort.rs | 8 ++++-- .../nextclade/src/sort/minimizer_index.rs | 4 +-- .../nextclade/src/sort/minimizer_search.rs | 10 ++++--- packages_rs/nextclade/src/sort/mod.rs | 1 + packages_rs/nextclade/src/sort/params.rs | 28 +++++++++++++++++++ 6 files changed, 49 insertions(+), 13 deletions(-) create mode 100644 packages_rs/nextclade/src/sort/params.rs diff --git a/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs b/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs index d8007488e..d67635e4b 100644 --- a/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs +++ b/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs @@ -12,6 +12,7 @@ use itertools::Itertools; use lazy_static::lazy_static; use nextclade::io::fs::add_extension; use nextclade::run::params::NextcladeInputParamsOptional; +use nextclade::sort::params::NextcladeSeqSortParams; use nextclade::utils::global_init::setup_logger; use nextclade::{getenv, make_error}; use std::fmt::Debug; @@ -671,6 +672,12 @@ pub struct NextcladeSeqSortArgs { #[clap(hide_long_help = true, hide_short_help = true)] pub output_dir: Option, + #[clap(flatten, next_help_heading = " Algorithm")] + pub search_params: NextcladeSeqSortParams, + + #[clap(flatten, next_help_heading = " Other")] + pub other_params: NextcladeRunOtherParams, + /// Use custom dataset server. /// /// You can host your own dataset server, with one or more datasets, grouped into dataset collections, and use this server to provide datasets to users of Nextclade CLI and Nextclade Web. Refer to Nextclade dataset documentation for more details. @@ -681,10 +688,6 @@ pub struct NextcladeSeqSortArgs { #[clap(flatten)] pub proxy_config: ProxyConfig, - - /// Number of processing jobs. If not specified, all available CPU threads will be used. - #[clap(global = false, long, short = 'j', default_value_t = num_cpus::get())] - pub jobs: usize, } fn generate_completions(shell: &str) -> Result<(), Report> { diff --git a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs index bb78f8f5b..eb7b1f4f5 100644 --- a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs +++ b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs @@ -1,4 +1,4 @@ -use crate::cli::nextclade_cli::NextcladeSeqSortArgs; +use crate::cli::nextclade_cli::{NextcladeRunOtherParams, NextcladeSeqSortArgs}; use crate::dataset::dataset_download::download_datasets_index_json; use crate::io::http_client::HttpClient; use eyre::{Report, WrapErr}; @@ -8,6 +8,7 @@ use nextclade::io::fasta::{FastaReader, FastaRecord}; use nextclade::make_error; use nextclade::sort::minimizer_index::{MinimizerIndexJson, MINIMIZER_INDEX_ALGO_VERSION}; use nextclade::sort::minimizer_search::{run_minimizer_search, MinimizerSearchResult}; +use nextclade::sort::params::NextcladeSeqSortParams; use nextclade::utils::string::truncate; #[derive(Debug, Clone)] @@ -65,7 +66,8 @@ pub fn run(args: &NextcladeSeqSortArgs, minimizer_index: &MinimizerIndexJson) -> let NextcladeSeqSortArgs { input_fastas, output_dir, - jobs, + search_params, + other_params: NextcladeRunOtherParams { jobs }, .. } = args; @@ -100,7 +102,7 @@ pub fn run(args: &NextcladeSeqSortArgs, minimizer_index: &MinimizerIndexJson) -> for fasta_record in &fasta_receiver { info!("Processing sequence '{}'", fasta_record.seq_name); - let result = run_minimizer_search(&fasta_record, minimizer_index) + let result = run_minimizer_search(&fasta_record, minimizer_index, search_params) .wrap_err_with(|| { format!( "When processing sequence #{} '{}'", diff --git a/packages_rs/nextclade/src/sort/minimizer_index.rs b/packages_rs/nextclade/src/sort/minimizer_index.rs index 144ff6262..291b88ce8 100644 --- a/packages_rs/nextclade/src/sort/minimizer_index.rs +++ b/packages_rs/nextclade/src/sort/minimizer_index.rs @@ -25,7 +25,7 @@ pub struct MinimizerIndexJson { pub version: String, - pub params: MinimizerParams, + pub params: MinimizerIndexParams, #[schemars(with = "BTreeMap")] #[serde(serialize_with = "serde_serialize_minimizers")] @@ -67,7 +67,7 @@ pub fn serde_deserialize_minimizers<'de, D: Deserializer<'de>>(deserializer: D) #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] -pub struct MinimizerParams { +pub struct MinimizerIndexParams { pub k: i64, pub cutoff: i64, diff --git a/packages_rs/nextclade/src/sort/minimizer_search.rs b/packages_rs/nextclade/src/sort/minimizer_search.rs index ce13e45a1..112ca1a08 100644 --- a/packages_rs/nextclade/src/sort/minimizer_search.rs +++ b/packages_rs/nextclade/src/sort/minimizer_search.rs @@ -1,5 +1,6 @@ use crate::io::fasta::FastaRecord; -use crate::sort::minimizer_index::{MinimizerIndexJson, MinimizerParams}; +use crate::sort::minimizer_index::{MinimizerIndexJson, MinimizerIndexParams}; +use crate::sort::params::NextcladeSeqSortParams; use eyre::Report; use itertools::Itertools; use schemars::JsonSchema; @@ -20,6 +21,7 @@ pub struct MinimizerSearchResult { pub fn run_minimizer_search( fasta_record: &FastaRecord, index: &MinimizerIndexJson, + params: &NextcladeSeqSortParams, ) -> Result { let normalization = &index.normalization; let n_refs = index.references.len(); @@ -43,7 +45,7 @@ pub fn run_minimizer_search( // require at least 30% of the maximal hits and at least 10 hits let max_normalized_hit = normalized_hits.iter().copied().fold(0.0, f64::max); let total_hits: u64 = hit_counts.iter().sum(); - if max_normalized_hit < 0.3 || total_hits < 10 { + if max_normalized_hit < params.min_normalized_hit || total_hits < params.min_total_hits { Ok(MinimizerSearchResult { dataset: None, hit_counts, @@ -79,7 +81,7 @@ const fn invertible_hash(x: u64) -> u64 { x } -fn get_hash(kmer: &[u8], params: &MinimizerParams) -> u64 { +fn get_hash(kmer: &[u8], params: &MinimizerIndexParams) -> u64 { let cutoff = params.cutoff as u64; let mut x = 0; @@ -111,7 +113,7 @@ fn get_hash(kmer: &[u8], params: &MinimizerParams) -> u64 { invertible_hash(x) } -pub fn get_ref_search_minimizers(seq: &FastaRecord, params: &MinimizerParams) -> Vec { +pub fn get_ref_search_minimizers(seq: &FastaRecord, params: &MinimizerIndexParams) -> Vec { let k = params.k as usize; let cutoff = params.cutoff as u64; diff --git a/packages_rs/nextclade/src/sort/mod.rs b/packages_rs/nextclade/src/sort/mod.rs index 8c3a0809b..c8b4c34db 100644 --- a/packages_rs/nextclade/src/sort/mod.rs +++ b/packages_rs/nextclade/src/sort/mod.rs @@ -1,2 +1,3 @@ pub mod minimizer_index; pub mod minimizer_search; +pub mod params; diff --git a/packages_rs/nextclade/src/sort/params.rs b/packages_rs/nextclade/src/sort/params.rs new file mode 100644 index 000000000..999f9c428 --- /dev/null +++ b/packages_rs/nextclade/src/sort/params.rs @@ -0,0 +1,28 @@ +use clap::Parser; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[allow(clippy::struct_excessive_bools)] +#[derive(Parser, Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct NextcladeSeqSortParams { + /// Minimum value of the normalized index hit being considered for assignment + #[clap(long)] + #[clap(default_value_t = NextcladeSeqSortParams::default().min_normalized_hit)] + pub min_normalized_hit: f64, + + /// Minimum number of the index hits required for assignment + #[clap(long)] + #[clap(default_value_t = NextcladeSeqSortParams::default().min_total_hits)] + pub min_total_hits: u64, +} + +#[allow(clippy::derivable_impls)] +impl Default for NextcladeSeqSortParams { + fn default() -> Self { + Self { + min_normalized_hit: 0.3, + min_total_hits: 10, + } + } +} From 5b6bc875bdb39c49a4007418770ad451caa3d24e Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Mon, 4 Sep 2023 17:33:33 +0200 Subject: [PATCH 07/51] fix(cli): allow minimizer index to be absent --- packages_rs/nextclade/src/io/dataset.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/packages_rs/nextclade/src/io/dataset.rs b/packages_rs/nextclade/src/io/dataset.rs index 402437a20..6169af6fb 100644 --- a/packages_rs/nextclade/src/io/dataset.rs +++ b/packages_rs/nextclade/src/io/dataset.rs @@ -21,6 +21,7 @@ pub struct DatasetsIndexJson { pub schema_version: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] pub minimizer_index: Vec, #[serde(flatten)] From 1b90f1b2b198db9f342e271dd69702036fd409d3 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Mon, 4 Sep 2023 18:28:14 +0200 Subject: [PATCH 08/51] feat(cli): write sorted fasta files --- Cargo.lock | 1 + packages_rs/nextclade-cli/Cargo.toml | 1 + .../nextclade-cli/src/cli/nextclade_cli.rs | 24 +++++ .../src/cli/nextclade_seq_sort.rs | 92 ++++++++++++++++++- 4 files changed, 115 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 44d8841e9..70a948561 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1810,6 +1810,7 @@ dependencies = [ "serde_json", "strum 0.25.0", "strum_macros 0.25.0", + "tinytemplate", "url", "zip", ] diff --git a/packages_rs/nextclade-cli/Cargo.toml b/packages_rs/nextclade-cli/Cargo.toml index 2b891e126..aa8366840 100644 --- a/packages_rs/nextclade-cli/Cargo.toml +++ b/packages_rs/nextclade-cli/Cargo.toml @@ -39,6 +39,7 @@ serde = { version = "=1.0.164", features = ["derive"] } serde_json = { version = "=1.0.99", features = ["preserve_order", "indexmap", "unbounded_depth"] } strum = "=0.25.0" strum_macros = "=0.25" +tinytemplate = "=1.2.1" url = { version = "=2.4.0", features = ["serde"] } zip = { version = "=0.6.6", default-features = false, features = ["aes-crypto", "bzip2", "deflate", "time"] } diff --git a/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs b/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs index d67635e4b..d82056a69 100644 --- a/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs +++ b/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs @@ -667,11 +667,35 @@ pub struct NextcladeSeqSortArgs { /// /// Sequences will be written in subdirectories: one subdirectory per dataset. Sequences inferred to be belonging to a particular dataset wil lbe places in the corresponding subdirectory. The subdirectory tree can be nested, depending on how dataset names are organized. /// + /// Mutually exclusive with `--output`. + /// #[clap(long)] #[clap(value_hint = ValueHint::DirPath)] #[clap(hide_long_help = true, hide_short_help = true)] + #[clap(group = "outputs")] pub output_dir: Option, + /// Template string for the file path to output sorted sequences. A separate file will be generated per dataset. + /// + /// The string should contain template variable `{name}`, where the dataset name will be substituted. Note that if the `{name}` variable contains slashes, they will be interpreted as path segments and subdirectories will be created. + /// + /// Make sure you properly quote and/or escape the curly braces, so that your shell, programming language or pipeline manager does not attempt to substitute the variables. + /// + /// Mutually exclusive with `--output-dir`. + /// + /// If the provided file path ends with one of the supported extensions: "gz", "bz2", "xz", "zst", then the file will be written compressed. + /// + /// If the required directory tree does not exist, it will be created. + /// + /// Example for bash shell: + /// + /// --output='outputs/{name}/sorted.fasta.gz' + #[clap(long)] + #[clap(value_hint = ValueHint::DirPath)] + #[clap(hide_long_help = true, hide_short_help = true)] + #[clap(group = "outputs")] + pub output: Option, + #[clap(flatten, next_help_heading = " Algorithm")] pub search_params: NextcladeSeqSortParams, diff --git a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs index eb7b1f4f5..8cdbe6a6a 100644 --- a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs +++ b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs @@ -3,13 +3,17 @@ use crate::dataset::dataset_download::download_datasets_index_json; use crate::io::http_client::HttpClient; use eyre::{Report, WrapErr}; use itertools::Itertools; -use log::{info, LevelFilter}; -use nextclade::io::fasta::{FastaReader, FastaRecord}; +use log::{info, trace, LevelFilter}; +use nextclade::io::fasta::{FastaReader, FastaRecord, FastaWriter}; use nextclade::make_error; use nextclade::sort::minimizer_index::{MinimizerIndexJson, MINIMIZER_INDEX_ALGO_VERSION}; use nextclade::sort::minimizer_search::{run_minimizer_search, MinimizerSearchResult}; -use nextclade::sort::params::NextcladeSeqSortParams; use nextclade::utils::string::truncate; +use serde::Serialize; +use std::collections::BTreeMap; +use std::path::PathBuf; +use std::str::FromStr; +use tinytemplate::TinyTemplate; #[derive(Debug, Clone)] struct MinimizerSearchRecord { @@ -18,6 +22,8 @@ struct MinimizerSearchRecord { } pub fn nextclade_seq_sort(args: &NextcladeSeqSortArgs) -> Result<(), Report> { + check_args(args)?; + let NextcladeSeqSortArgs { server, proxy_config, @@ -66,6 +72,7 @@ pub fn run(args: &NextcladeSeqSortArgs, minimizer_index: &MinimizerIndexJson) -> let NextcladeSeqSortArgs { input_fastas, output_dir, + output, search_params, other_params: NextcladeRunOtherParams { jobs }, .. @@ -122,11 +129,55 @@ pub fn run(args: &NextcladeSeqSortArgs, minimizer_index: &MinimizerIndexJson) -> } let writer = s.spawn(move || { + let output_dir = &output_dir; + let output = &output; + + let tt = output.as_ref().map(move |output| { + let mut tt = TinyTemplate::new(); + tt.add_template("output", output) + .wrap_err_with(|| format!("When parsing template: {output}")) + .unwrap(); + tt + }); + println!( "{:40} | {:40} | {:10} | {:10}", "Seq. name", "dataset", "total hits", "max hit" ); + + let mut writers = BTreeMap::new(); + for record in result_receiver { + if let Some(name) = &record.result.dataset { + let filepath = match (&tt, output_dir) { + (Some(tt), None) => { + let filepath_str = tt + .render("output", &OutputTemplateContext { name }) + .wrap_err("When rendering output path template") + .unwrap(); + + Some( + PathBuf::from_str(&filepath_str) + .wrap_err_with(|| format!("Invalid output translations path: '{filepath_str}'")) + .unwrap(), + ) + } + (None, Some(output_dir)) => Some(output_dir.join(name).join("sequences.fasta")), + _ => None, + }; + + if let Some(filepath) = filepath { + let writer = writers.entry(filepath.clone()).or_insert_with(|| { + trace!("Creating fasta writer to file {filepath:#?}"); + FastaWriter::from_path(filepath).unwrap() + }); + + writer + .write(&record.fasta_record.seq_name, &record.fasta_record.seq, false) + .unwrap(); + } + } + println!( "{:40} | {:40} | {:>10} | {:>.3}", &truncate(record.fasta_record.seq_name, 40), @@ -140,3 +191,38 @@ pub fn run(args: &NextcladeSeqSortArgs, minimizer_index: &MinimizerIndexJson) -> Ok(()) } + +#[derive(Serialize)] +struct OutputTemplateContext<'a> { + name: &'a str, +} + +fn check_args(args: &NextcladeSeqSortArgs) -> Result<(), Report> { + let NextcladeSeqSortArgs { output_dir, output, .. } = args; + + if output.is_some() && output_dir.is_some() { + return make_error!( + "The arguments `--output-dir` and `--output` cannot be used together. Remove one or the other." + ); + } + + if let Some(output) = output { + if !output.contains("{name}") { + return make_error!( + r#" +Expected `--output` argument to contain a template string containing template variable {{name}} (with curly braces), but received: + + {output} + +Make sure the variable is not substituted by your shell, programming language or workflow manager. Apply proper escaping as needed. +Example for bash shell: + + --output='outputs/{{name}}/sorted.fasta.gz' + + "# + ); + } + } + + Ok(()) +} From db37dd3985efa55c8a806613e3c8145be59f6c0c Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Mon, 4 Sep 2023 20:11:23 +0200 Subject: [PATCH 09/51] fix(web): dataset equality --- packages_rs/nextclade-web/src/types.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages_rs/nextclade-web/src/types.ts b/packages_rs/nextclade-web/src/types.ts index 7e8a7b50a..6cabb3df4 100644 --- a/packages_rs/nextclade-web/src/types.ts +++ b/packages_rs/nextclade-web/src/types.ts @@ -1,4 +1,4 @@ -import { isEqual, isNil, range, sumBy } from 'lodash' +import { isNil, range, sumBy } from 'lodash' import type { Aa, Cds, @@ -135,5 +135,5 @@ export interface AlgorithmInput { } export function areDatasetsEqual(left?: Dataset, right?: Dataset): boolean { - return !isNil(left) && !isNil(right) && isEqual(left.attributes, right.attributes) + return !isNil(left?.path) && !isNil(right?.path) && left?.path === right?.path } From 8ac3f679dac1bf78d1f3498c1c027f78ebd9ba75 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Tue, 5 Sep 2023 02:32:25 +0200 Subject: [PATCH 10/51] feat(web): add sequence sorting prototype --- .../src/cli/nextclade_seq_sort.rs | 8 +- packages_rs/nextclade-web/src/build.rs | 5 ++ .../components/Autodetect/AutodetectPage.tsx | 89 +++++++++++++++++++ .../src/components/Main/DatasetCurrent.tsx | 44 ++++----- .../src/components/Main/DatasetInfo.tsx | 24 +++++ .../components/Main/DatasetSelectorList.tsx | 22 +++++ .../Main/MainInputFormSequenceFilePicker.tsx | 16 +++- .../src/hooks/useRunSeqAutodetect.ts | 58 ++++++++++++ .../nextclade-web/src/io/fetchDatasets.ts | 24 +++-- .../src/io/fetchDatasetsIndex.ts | 28 +++++- packages_rs/nextclade-web/src/lib.rs | 8 ++ packages_rs/nextclade-web/src/pages/_app.tsx | 21 +++-- .../nextclade-web/src/pages/autodetect.tsx | 1 + .../src/state/autodetect.state.ts | 68 ++++++++++++++ .../nextclade-web/src/state/dataset.state.ts | 12 ++- packages_rs/nextclade-web/src/wasm/jserr.rs | 8 ++ packages_rs/nextclade-web/src/wasm/main.rs | 12 +-- packages_rs/nextclade-web/src/wasm/mod.rs | 2 + .../nextclade-web/src/wasm/seq_autodetect.rs | 59 ++++++++++++ .../src/workers/launchAnalysis.ts | 2 +- .../src/workers/nextcladeAutodetect.worker.ts | 60 +++++++++++++ .../src/workers/nextcladeWasm.worker.ts | 4 +- .../nextclade/src/sort/minimizer_search.rs | 7 ++ 23 files changed, 520 insertions(+), 62 deletions(-) create mode 100644 packages_rs/nextclade-web/src/components/Autodetect/AutodetectPage.tsx create mode 100644 packages_rs/nextclade-web/src/hooks/useRunSeqAutodetect.ts create mode 100644 packages_rs/nextclade-web/src/pages/autodetect.tsx create mode 100644 packages_rs/nextclade-web/src/state/autodetect.state.ts create mode 100644 packages_rs/nextclade-web/src/wasm/jserr.rs create mode 100644 packages_rs/nextclade-web/src/wasm/seq_autodetect.rs create mode 100644 packages_rs/nextclade-web/src/workers/nextcladeAutodetect.worker.ts diff --git a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs index 8cdbe6a6a..670537bf5 100644 --- a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs +++ b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs @@ -7,7 +7,7 @@ use log::{info, trace, LevelFilter}; use nextclade::io::fasta::{FastaReader, FastaRecord, FastaWriter}; use nextclade::make_error; use nextclade::sort::minimizer_index::{MinimizerIndexJson, MINIMIZER_INDEX_ALGO_VERSION}; -use nextclade::sort::minimizer_search::{run_minimizer_search, MinimizerSearchResult}; +use nextclade::sort::minimizer_search::{run_minimizer_search, MinimizerSearchRecord}; use nextclade::utils::string::truncate; use serde::Serialize; use std::collections::BTreeMap; @@ -15,12 +15,6 @@ use std::path::PathBuf; use std::str::FromStr; use tinytemplate::TinyTemplate; -#[derive(Debug, Clone)] -struct MinimizerSearchRecord { - pub fasta_record: FastaRecord, - pub result: MinimizerSearchResult, -} - pub fn nextclade_seq_sort(args: &NextcladeSeqSortArgs) -> Result<(), Report> { check_args(args)?; diff --git a/packages_rs/nextclade-web/src/build.rs b/packages_rs/nextclade-web/src/build.rs index 9d6b85f9e..6dcd7284b 100644 --- a/packages_rs/nextclade-web/src/build.rs +++ b/packages_rs/nextclade-web/src/build.rs @@ -16,6 +16,8 @@ use nextclade::qc::qc_run::QcResult; use nextclade::run::nextclade_wasm::{ AnalysisInitialData, AnalysisInput, NextcladeParams, NextcladeParamsRaw, NextcladeResult, OutputTrees, }; +use nextclade::sort::minimizer_index::MinimizerIndexJson; +use nextclade::sort::minimizer_search::{MinimizerSearchRecord, MinimizerSearchResult}; use nextclade::translate::translate_genes::Translation; use nextclade::tree::tree::{AuspiceTree, CladeNodeAttrKeyDesc}; use nextclade::types::outputs::{NextcladeErrorOutputs, NextcladeOutputs}; @@ -73,4 +75,7 @@ struct _SchemaRoot<'a> { _27: DatasetAttributeValue, _28: DatasetAttributes, _29: DatasetCollectionUrl, + _30: MinimizerIndexJson, + _31: MinimizerSearchResult, + _32: MinimizerSearchRecord, } diff --git a/packages_rs/nextclade-web/src/components/Autodetect/AutodetectPage.tsx b/packages_rs/nextclade-web/src/components/Autodetect/AutodetectPage.tsx new file mode 100644 index 000000000..122620cf8 --- /dev/null +++ b/packages_rs/nextclade-web/src/components/Autodetect/AutodetectPage.tsx @@ -0,0 +1,89 @@ +import React from 'react' +import { Col as ColBase, Row as RowBase } from 'reactstrap' +import { useRecoilValue } from 'recoil' +import { TableSlimWithBorders } from 'src/components/Common/TableSlim' +import { Layout } from 'src/components/Layout/Layout' +import { useTranslationSafe } from 'src/helpers/useTranslationSafe' +import { autodetectResultsAtom } from 'src/state/autodetect.state' +import styled from 'styled-components' + +const Container = styled.div` + margin-top: 1rem; + height: 100%; + overflow: hidden; +` + +const Row = styled(RowBase)` + overflow: hidden; + height: 100%; +` + +const Col = styled(ColBase)` + overflow: hidden; + height: 100%; +` + +const Table = styled(TableSlimWithBorders)` + padding-top: 50px; + + & thead { + height: 51px; + position: sticky; + top: -2px; + background-color: ${(props) => props.theme.gray700}; + color: ${(props) => props.theme.gray100}; + } + + & thead th { + margin: auto; + text-align: center; + vertical-align: middle; + } +` + +const TableWrapper = styled.div` + height: 100%; + overflow-y: auto; +` + +export function AutodetectPage() { + const { t } = useTranslationSafe() + // const minimizerIndex = useRecoilValue(minimizerIndexAtom) + const autodetectResults = useRecoilValue(autodetectResultsAtom) + + return ( + + + + + + + + + + + + + + + + + + {autodetectResults.map((res) => ( + + + + + + + + ))} + +
{'#'}{t('Seq. name')}{t('dataset')}{t('total hits')}{t('max hit')}
{res.fastaRecord.index}{res.fastaRecord.seqName}{res.result.dataset ?? ''}{res.result.totalHits}{res.result.maxNormalizedHit.toFixed(3)}
+ + + + + + ) +} diff --git a/packages_rs/nextclade-web/src/components/Main/DatasetCurrent.tsx b/packages_rs/nextclade-web/src/components/Main/DatasetCurrent.tsx index af0807201..0c4d5167e 100644 --- a/packages_rs/nextclade-web/src/components/Main/DatasetCurrent.tsx +++ b/packages_rs/nextclade-web/src/components/Main/DatasetCurrent.tsx @@ -1,5 +1,5 @@ import { isNil } from 'lodash' -import React, { useCallback, useState } from 'react' +import React, { useCallback, useMemo, useState } from 'react' import { Button, Col, Collapse, Row, UncontrolledAlert } from 'reactstrap' import { useRecoilState, useRecoilValue, useResetRecoilState, useSetRecoilState } from 'recoil' import styled from 'styled-components' @@ -82,6 +82,28 @@ export function DatasetCurrent() { const onCustomizeClicked = useCallback(() => setAdvancedOpen((advancedOpen) => !advancedOpen), []) + const customize = useMemo(() => { + if (datasetCurrent?.path === 'autodetect') { + return null + } + + return ( + + + + + + + + + + + + + + ) + }, [advancedOpen, datasetCurrent?.path, onCustomizeClicked]) + if (!datasetCurrent) { return null } @@ -105,29 +127,11 @@ export function DatasetCurrent() { {t('Change')} - - {t('Recent dataset updates')} - - - - - - - - - - - - - - + {customize} ) diff --git a/packages_rs/nextclade-web/src/components/Main/DatasetInfo.tsx b/packages_rs/nextclade-web/src/components/Main/DatasetInfo.tsx index 0ea36038c..adb5af26c 100644 --- a/packages_rs/nextclade-web/src/components/Main/DatasetInfo.tsx +++ b/packages_rs/nextclade-web/src/components/Main/DatasetInfo.tsx @@ -21,6 +21,11 @@ export const DatasetInfoLine = styled.p` font-size: 0.9rem; padding: 0; margin: 0; + + &:after { + content: ' '; + white-space: pre; + } ` const DatasetInfoBadge = styled(Badge)` @@ -50,6 +55,10 @@ export function DatasetInfo({ dataset }: DatasetInfoProps) { return null } + if (path === 'autodetect') { + return + } + return ( @@ -107,3 +116,18 @@ export function DatasetInfo({ dataset }: DatasetInfoProps) { ) } + +export function DatasetAutodetectInfo() { + const { t } = useTranslationSafe() + + return ( + + + {t('Autodetect')} + + {t('Detect pathogen automatically from sequences')} + + + + ) +} diff --git a/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx b/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx index 6ccdafb79..d2f148b68 100644 --- a/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx +++ b/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx @@ -18,6 +18,20 @@ import { DatasetInfo } from 'src/components/Main/DatasetInfo' // border-radius: 5px; // ` +const DATASET_AUTODETECT: Dataset = { + path: 'autodetect', + enabled: true, + official: true, + attributes: { + name: { value: 'autodetect', valueFriendly: 'Autodetect' }, + reference: { value: 'autodetect', valueFriendly: 'Autodetect' }, + }, + files: { + reference: '', + pathogenJson: '', + }, +} + export const DatasetSelectorUl = styled(ListGroup)` flex: 1; overflow-y: scroll; @@ -79,6 +93,14 @@ export function DatasetSelectorList({ return ( // + { + + } + {[itemsStartWith, itemsInclude].map((datasets) => datasets.map((dataset) => ( , []) - const run = useRunAnalysis() + const runAnalysis = useRunAnalysis() + const runAutodetect = useRunSeqAutodetect() + + const run = useCallback(() => { + if (datasetCurrent?.path === 'autodetect') { + runAutodetect() + } else { + runAnalysis() + } + }, [datasetCurrent?.path, runAnalysis, runAutodetect]) const setSequences = useCallback( (inputs: AlgorithmInput[]) => { addQryInputs(inputs) - if (shouldRunAutomatically) { run() } @@ -62,7 +71,6 @@ export function MainInputFormSequenceFilePicker() { const setExampleSequences = useCallback(() => { if (datasetCurrent) { addQryInputs([new AlgorithmInputDefault(datasetCurrent)]) - if (shouldRunAutomatically) { run() } @@ -81,7 +89,7 @@ export function MainInputFormSequenceFilePicker() { }, [canRun, hasInputErrors, hasRequiredInputs, t]) const LoadExampleLink = useMemo(() => { - const cannotLoadExample = hasInputErrors || !datasetCurrent + const cannotLoadExample = hasInputErrors || !datasetCurrent || datasetCurrent.path === 'autodetect' return ( - - - + {t('Suggest best matches')} + + + + + + ) } diff --git a/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx b/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx index b9049fb7d..dc82ac797 100644 --- a/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx +++ b/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx @@ -1,40 +1,15 @@ -import { isNil } from 'lodash' +import { isNil, sortBy } from 'lodash' import React, { useCallback, useMemo } from 'react' - import { ListGroup, ListGroupItem } from 'reactstrap' import { useRecoilValue } from 'recoil' -import { minimizerIndexVersionAtom } from 'src/state/dataset.state' import styled from 'styled-components' - import type { Dataset } from 'src/types' import { areDatasetsEqual } from 'src/types' -import { search } from 'src/helpers/search' +import { autodetectResultsAtom } from 'src/state/autodetect.state' +// import { datasetsAtom } from 'src/state/dataset.state' +// import { search } from 'src/helpers/search' import { DatasetInfo } from 'src/components/Main/DatasetInfo' -// export const DatasetSelectorContainer = styled.div` -// flex: 1 0 100%; -// display: flex; -// flex-direction: column; -// overflow: hidden; -// height: 100%; -// border: 1px #ccc solid; -// border-radius: 5px; -// ` - -const DATASET_AUTODETECT: Dataset = { - path: 'autodetect', - enabled: true, - official: true, - attributes: { - name: { value: 'autodetect', valueFriendly: 'Autodetect' }, - reference: { value: 'autodetect', valueFriendly: 'Autodetect' }, - }, - files: { - reference: '', - pathogenJson: '', - }, -} - export const DatasetSelectorUl = styled(ListGroup)` flex: 1; overflow-y: scroll; @@ -75,45 +50,46 @@ export interface DatasetSelectorListProps { export function DatasetSelectorList({ datasets, - searchTerm, + // searchTerm, datasetHighlighted, onDatasetHighlighted, }: DatasetSelectorListProps) { - const minimizerIndexVersion = useRecoilValue(minimizerIndexVersionAtom) - const onItemClick = useCallback((dataset: Dataset) => () => onDatasetHighlighted(dataset), [onDatasetHighlighted]) - const autodetectItem = useMemo(() => { - if (isNil(minimizerIndexVersion)) { - return null + const autodetectResults = useRecoilValue(autodetectResultsAtom) + + const { itemsStartWith, itemsInclude, itemsNotInclude } = useMemo(() => { + if (isNil(autodetectResults) || autodetectResults.length === 0) { + return { itemsStartWith: [], itemsInclude: datasets, itemsNotInclude: [] } } - return ( - + let itemsInclude = datasets.filter((candidate) => + autodetectResults.some((result) => result.result.dataset === candidate.path), + ) + itemsInclude = sortBy( + itemsInclude, + (dataset) => -autodetectResults.filter((result) => result.result.dataset === dataset.path).length, ) - }, [datasetHighlighted, minimizerIndexVersion, onItemClick]) - const { itemsStartWith, itemsInclude, itemsNotInclude } = useMemo(() => { - if (searchTerm.trim().length === 0) { - return { itemsStartWith: datasets, itemsInclude: [], itemsNotInclude: [] } - } + const itemsNotInclude = datasets.filter((candidate) => !itemsInclude.map((it) => it.path).includes(candidate.path)) + + return { itemsStartWith: [], itemsInclude, itemsNotInclude } + }, [autodetectResults, datasets]) - return search(datasets, searchTerm, (dataset) => [ - dataset.attributes.name.value, - dataset.attributes.name.valueFriendly ?? '', - dataset.attributes.reference.value, - ]) - }, [datasets, searchTerm]) + // const { itemsStartWith, itemsInclude, itemsNotInclude } = useMemo(() => { + // if (searchTerm.trim().length === 0) { + // return { itemsStartWith: datasets, itemsInclude: [], itemsNotInclude: [] } + // } + // + // return search(datasets, searchTerm, (dataset) => [ + // dataset.attributes.name.value, + // dataset.attributes.name.valueFriendly ?? '', + // dataset.attributes.reference.value, + // ]) + // }, [datasets, searchTerm]) return ( - // - {autodetectItem} - {[itemsStartWith, itemsInclude].map((datasets) => datasets.map((dataset) => ( - // ) } diff --git a/packages_rs/nextclade-web/src/components/Main/MainInputForm.tsx b/packages_rs/nextclade-web/src/components/Main/MainInputForm.tsx index 165973665..c3c0a890b 100644 --- a/packages_rs/nextclade-web/src/components/Main/MainInputForm.tsx +++ b/packages_rs/nextclade-web/src/components/Main/MainInputForm.tsx @@ -1,11 +1,8 @@ -import React, { useMemo, useState } from 'react' -import { useRecoilValue } from 'recoil' +import React from 'react' +import { QuerySequenceFilePicker } from 'src/components/Main/QuerySequenceFilePicker' import styled from 'styled-components' import { Col as ColBase, Row as RowBase } from 'reactstrap' -import { DatasetContentSection } from 'src/components/Main/DatasetContentSection' import { DatasetSelector } from 'src/components/Main/DatasetSelector' -import { MainInputFormRunStep } from 'src/components/Main/MainInputFormRunStep' -import { datasetCurrentAtom } from 'src/state/dataset.state' import { useUpdatedDatasetIndex } from 'src/io/fetchDatasets' const Container = styled.div` @@ -24,29 +21,18 @@ const Col = styled(ColBase)` ` export function MainInputForm() { - const [searchTerm, setSearchTerm] = useState('') - const currentDataset = useRecoilValue(datasetCurrentAtom) - // This periodically fetches dataset index and updates the list of datasets. useUpdatedDatasetIndex() - const FormBody = useMemo( - () => - currentDataset ? ( - - ) : ( - - ), - [currentDataset, searchTerm], - ) - return ( - + + + + - {FormBody} ) diff --git a/packages_rs/nextclade-web/src/components/Main/MainInputFormRunStep.tsx b/packages_rs/nextclade-web/src/components/Main/MainInputFormRunStep.tsx deleted file mode 100644 index f06d76b6d..000000000 --- a/packages_rs/nextclade-web/src/components/Main/MainInputFormRunStep.tsx +++ /dev/null @@ -1,35 +0,0 @@ -import React from 'react' - -import { Col, Container, Row } from 'reactstrap' -import styled from 'styled-components' - -import { MainInputFormSequenceFilePicker } from 'src/components/Main/MainInputFormSequenceFilePicker' -import { DatasetCurrent } from './DatasetCurrent' - -const MainInputFormContainer = styled(Container)` - display: flex; - flex-direction: column; - width: 100%; - height: 100%; - overflow: hidden; - margin: 0; - padding: 0; -` - -export function MainInputFormRunStep() { - return ( - - - - - - - - - - - - - - ) -} diff --git a/packages_rs/nextclade-web/src/components/Main/MainInputFormSequenceFilePicker.tsx b/packages_rs/nextclade-web/src/components/Main/QuerySequenceFilePicker.tsx similarity index 73% rename from packages_rs/nextclade-web/src/components/Main/MainInputFormSequenceFilePicker.tsx rename to packages_rs/nextclade-web/src/components/Main/QuerySequenceFilePicker.tsx index afc8a2e8c..8eed4018f 100644 --- a/packages_rs/nextclade-web/src/components/Main/MainInputFormSequenceFilePicker.tsx +++ b/packages_rs/nextclade-web/src/components/Main/QuerySequenceFilePicker.tsx @@ -1,16 +1,15 @@ -import { noop } from 'lodash' import React, { useCallback, useMemo } from 'react' import { Button, Col, Form, FormGroup, Row } from 'reactstrap' -import { useRecoilState, useRecoilValue } from 'recoil' -import { MainInputFormSequencesCurrent } from 'src/components/Main/MainInputFormSequencesCurrent' +import { useRecoilValue } from 'recoil' +import { QuerySequenceList } from 'src/components/Main/QuerySequenceList' import { useRunAnalysis } from 'src/hooks/useRunAnalysis' import { useRunSeqAutodetect } from 'src/hooks/useRunSeqAutodetect' +import { useRecoilToggle } from 'src/hooks/useToggle' import { canRunAtom } from 'src/state/results.state' import styled from 'styled-components' - import { datasetCurrentAtom } from 'src/state/dataset.state' import { hasInputErrorsAtom, qrySeqErrorAtom } from 'src/state/error.state' -import { shouldRunAutomaticallyAtom } from 'src/state/settings.state' +import { shouldRunAutomaticallyAtom, shouldSuggestDatasetsAtom } from 'src/state/settings.state' import type { AlgorithmInput } from 'src/types' import { Toggle } from 'src/components/Common/Toggle' import { FlexLeft, FlexRight } from 'src/components/FilePicker/FilePickerStyles' @@ -33,7 +32,7 @@ const ButtonRunStyled = styled(Button)` margin-left: 1rem; ` -export function MainInputFormSequenceFilePicker() { +export function QuerySequenceFilePicker() { const { t } = useTranslationSafe() const datasetCurrent = useRecoilValue(datasetCurrentAtom) @@ -41,7 +40,9 @@ export function MainInputFormSequenceFilePicker() { const qrySeqError = useRecoilValue(qrySeqErrorAtom) const canRun = useRecoilValue(canRunAtom) - const [shouldRunAutomatically, setShouldRunAutomatically] = useRecoilState(shouldRunAutomaticallyAtom) + const { state: shouldRunAutomatically, toggle: toggleRunAutomatically } = useRecoilToggle(shouldRunAutomaticallyAtom) + const shouldSuggestDatasets = useRecoilValue(shouldSuggestDatasetsAtom) + const hasRequiredInputs = useRecoilValue(hasRequiredInputsAtom) const hasInputErrors = useRecoilValue(hasInputErrorsAtom) @@ -50,32 +51,30 @@ export function MainInputFormSequenceFilePicker() { const runAnalysis = useRunAnalysis() const runAutodetect = useRunSeqAutodetect() - const run = useCallback(() => { - if (datasetCurrent?.path === 'autodetect') { - runAutodetect() - } else { - runAnalysis() - } - }, [datasetCurrent?.path, runAnalysis, runAutodetect]) - const setSequences = useCallback( (inputs: AlgorithmInput[]) => { addQryInputs(inputs) + if (shouldSuggestDatasets) { + runAutodetect() + } if (shouldRunAutomatically) { - run() + runAnalysis() } }, - [addQryInputs, run, shouldRunAutomatically], + [addQryInputs, runAnalysis, runAutodetect, shouldRunAutomatically, shouldSuggestDatasets], ) const setExampleSequences = useCallback(() => { if (datasetCurrent) { addQryInputs([new AlgorithmInputDefault(datasetCurrent)]) + if (shouldSuggestDatasets) { + runAutodetect() + } if (shouldRunAutomatically) { - run() + runAnalysis() } } - }, [addQryInputs, datasetCurrent, run, shouldRunAutomatically]) + }, [addQryInputs, datasetCurrent, runAnalysis, runAutodetect, shouldRunAutomatically, shouldSuggestDatasets]) const { isRunButtonDisabled, runButtonColor, runButtonTooltip } = useMemo(() => { const isRunButtonDisabled = !(canRun && hasRequiredInputs) || hasInputErrors @@ -88,19 +87,6 @@ export function MainInputFormSequenceFilePicker() { } }, [canRun, hasInputErrors, hasRequiredInputs, t]) - const LoadExampleLink = useMemo(() => { - const cannotLoadExample = hasInputErrors || !datasetCurrent || datasetCurrent.path === 'autodetect' - return ( - - ) - }, [datasetCurrent, hasInputErrors, setExampleSequences, t]) - - const onToggleRunAutomatically = useCallback(() => { - setShouldRunAutomatically((shouldRunAutomatically) => !shouldRunAutomatically) - }, [setShouldRunAutomatically]) - const headerText = useMemo(() => { if (qryInputs.length > 0) { return t('Add more sequence data') @@ -110,7 +96,7 @@ export function MainInputFormSequenceFilePicker() { return ( - + @@ -134,7 +119,7 @@ export function MainInputFormSequenceFilePicker() { {t('Run automatically')} @@ -145,12 +130,14 @@ export function MainInputFormSequenceFilePicker() { - {LoadExampleLink} + {t('Run')} diff --git a/packages_rs/nextclade-web/src/components/Main/MainInputFormSequencesCurrent.tsx b/packages_rs/nextclade-web/src/components/Main/QuerySequenceList.tsx similarity index 98% rename from packages_rs/nextclade-web/src/components/Main/MainInputFormSequencesCurrent.tsx rename to packages_rs/nextclade-web/src/components/Main/QuerySequenceList.tsx index feef13229..d7ec69ab5 100644 --- a/packages_rs/nextclade-web/src/components/Main/MainInputFormSequencesCurrent.tsx +++ b/packages_rs/nextclade-web/src/components/Main/QuerySequenceList.tsx @@ -46,7 +46,7 @@ export function InputFileInfo({ input, index }: InputFileInfoProps) { ) } -export function MainInputFormSequencesCurrent() { +export function QuerySequenceList() { const { t } = useTranslationSafe() const { qryInputs, clearQryInputs } = useQuerySeqInputs() diff --git a/packages_rs/nextclade-web/src/helpers/colorHash.ts b/packages_rs/nextclade-web/src/helpers/colorHash.ts new file mode 100644 index 000000000..79c91fad5 --- /dev/null +++ b/packages_rs/nextclade-web/src/helpers/colorHash.ts @@ -0,0 +1,215 @@ +/* eslint-disable no-param-reassign,no-plusplus,no-loops/no-loops,prefer-destructuring,no-else-return,unicorn/prefer-code-point */ + +/** + * Color Hash + * by Simone Piccian (zanza00) + * taken with modifications from + * https://github.com/zanza00/color-hash/blob/6d43fa1b103fa090e1f0d788f5bfc4e99bf02263/src/color-hash.ts + */ + +/** + * BKDR Hash (modified version) + * + * @param {String} str string to hash + * @returns {Number} + */ +export function BKDRHash(str: string): number { + const seed = 131 + const seed2 = 137 + let hash = 0 + // make hash more sensitive for short string like 'a', 'b', 'c' + str += 'x' + const MAX_SAFE_INTEGER = Number.MAX_SAFE_INTEGER / seed2 + for (let i = 0; i < str.length; i++) { + if (hash > MAX_SAFE_INTEGER) { + hash = Math.floor(hash / seed2) + } + hash = hash * seed + str.charCodeAt(i) + } + return hash +} + +/** + * Convert RGB Array to HEX + * + * @param {Array} RGBArray - [R, G, B] + * @returns {String} 6 digits hex starting with # + */ +function RGB2HEX(RGBArray: [number, number, number]): string { + let hex = '#' + RGBArray.forEach((value) => { + if (value < 16) { + hex += 0 + } + hex += value.toString(16) + }) + return hex +} + +type func = (p: number, q: number) => (color: number) => number +const paramToColor: func = (p, q) => (color) => { + if (color < 0) { + color++ + } + if (color > 1) { + color-- + } + if (color < 1 / 6) { + color = p + (q - p) * 6 * color + } else if (color < 0.5) { + color = q + } else if (color < 2 / 3) { + color = p + (q - p) * 6 * (2 / 3 - color) + } else { + color = p + } + return Math.round(color * 255) +} + +/** + * Convert HSL to RGB + * + * @see {@link https://en.wikipedia.org/wiki/HSL_and_HSV} for further information. + * @param {Number} H Hue ∈ [0, 360) + * @param {Number} S Saturation ∈ [0, 1] + * @param {Number} L Lightness ∈ [0, 1] + * @returns {Array} R, G, B ∈ [0, 255] + */ +function HSL2RGB(H: number, S: number, L: number): [number, number, number] { + const H360 = H / 360 + + const q = L < 0.5 ? L * (1 + S) : L + S - L * S + const p = 2 * L - q + + const partial = paramToColor(p, q) + + return [partial(H360 + 1 / 3), partial(H360), partial(H360 - 1 / 3)] +} + +export { HSL2RGB as testFroHSL2RGB } + +export type Options = { + lightness?: number | number[] + saturation?: number | number[] + hue?: number | { min: number; max: number } | { min: number; max: number }[] + hash?: typeof BKDRHash +} + +/** + * Color Hash Class + * + * @class + */ +class ColorHash { + private L: number[] + private S: number[] + private hueRanges: { min: number; max: number }[] + private hash: (str: string) => number + + constructor(options: Options = {}) { + const LS = [options.lightness ?? [0.35, 0.5, 0.65], options.saturation ?? [0.35, 0.5, 0.65]].map((param) => { + return Array.isArray(param) ? param.concat() : [param] + }) + + this.L = LS[0] + this.S = LS[1] + + if (typeof options.hue === 'number') { + options.hue = { min: options.hue, max: options.hue } + } + if (typeof options.hue === 'object' && !Array.isArray(options.hue)) { + options.hue = [options.hue] + } + if (typeof options.hue === 'undefined') { + options.hue = [] + } + this.hueRanges = options.hue.map((range) => { + return { + min: typeof range.min === 'undefined' ? 0 : range.min, + max: typeof range.max === 'undefined' ? 360 : range.max, + } + }) + + this.hash = options.hash ?? BKDRHash + } + + private getHue(hash: number): number { + if (this.hueRanges.length > 0) { + const range = this.hueRanges[hash % this.hueRanges.length] + const hueResolution = 727 // note that 727 is a prime + return (((hash / this.hueRanges.length) % hueResolution) * (range.max - range.min)) / hueResolution + range.min + } else { + return hash % 359 // note that 359 is a prime + } + } + + /** + * Returns the hash in [h, s, l]. + * Note that H ∈ [0, 360); S ∈ [0, 1]; L ∈ [0, 1]; + * + * @param {String} str string to hash + * @returns {Array} [h, s, l] + */ + hsl(str: string): [number, number, number] { + const hash = this.hash(str) + + const H = this.getHue(hash) + + const sHash = Math.floor(hash / 360) + + const S = this.S[sHash % this.S.length] + + const lHash = Math.floor(sHash / this.S.length) + + const L = this.L[lHash % this.L.length] + + return [H, S, L] + } + + /** + * Returns the hash in [r, g, b]. + * Note that R, G, B ∈ [0, 255] + * + * @param {String} str string to hash + * @returns {Array} [r, g, b] + */ + rgb(str: string): [number, number, number] { + const hsl = this.hsl(str) + return HSL2RGB(...hsl) + } + + /** + * Returns the hash in hex + * + * @param {String} str string to hash + * @returns {String} hex with # + */ + hex(str: string): string { + const rgb = this.rgb(str) + return RGB2HEX(rgb) + } +} + +export interface ColorHashOptions extends Options { + reverse?: boolean + prefix?: string + suffix?: string +} + +export function colorHash(content: string, options?: ColorHashOptions) { + let contentModified = content + + if (options?.reverse) { + contentModified = contentModified.split('').reverse().join('') + } + + if (options?.prefix) { + contentModified = `${options.prefix}${contentModified}` + } + + if (options?.suffix) { + contentModified = `${contentModified}${options.suffix}` + } + + return new ColorHash(options).hex(contentModified) +} diff --git a/packages_rs/nextclade-web/src/helpers/string.ts b/packages_rs/nextclade-web/src/helpers/string.ts index 47accaea3..531c4fdfb 100644 --- a/packages_rs/nextclade-web/src/helpers/string.ts +++ b/packages_rs/nextclade-web/src/helpers/string.ts @@ -41,3 +41,7 @@ export function findSimilarStrings(haystack: string[], needle: string): string[] scores = sortBy(scores, ({ score }) => -score) return scores.map(({ candidate }) => candidate) } + +export function firstLetter(s: string): string | undefined { + return s.split('').find((c) => c.match(/[A-z]/)) +} diff --git a/packages_rs/nextclade-web/src/hooks/useRunSeqAutodetect.ts b/packages_rs/nextclade-web/src/hooks/useRunSeqAutodetect.ts index 21d955c89..80ae6590c 100644 --- a/packages_rs/nextclade-web/src/hooks/useRunSeqAutodetect.ts +++ b/packages_rs/nextclade-web/src/hooks/useRunSeqAutodetect.ts @@ -1,29 +1,24 @@ -import { useRouter } from 'next/router' +import type { Subscription } from 'observable-fns' import { useRecoilCallback } from 'recoil' import { ErrorInternal } from 'src/helpers/ErrorInternal' import { axiosFetch } from 'src/io/axiosFetch' -import { autodetectResultAtom, autodetectResultsAtom, minimizerIndexAtom } from 'src/state/autodetect.state' +import { autodetectResultByIndexAtom, autodetectResultsAtom, minimizerIndexAtom } from 'src/state/autodetect.state' import { minimizerIndexVersionAtom } from 'src/state/dataset.state' import { MinimizerIndexJson, MinimizerSearchRecord } from 'src/types' import { qrySeqInputsStorageAtom } from 'src/state/inputs.state' import { getQueryFasta } from 'src/workers/launchAnalysis' -import type { Subscription } from 'observable-fns' import { NextcladeSeqAutodetectWasmWorker } from 'src/workers/nextcladeAutodetect.worker' import { spawn } from 'src/workers/spawn' export function useRunSeqAutodetect() { - const router = useRouter() - return useRecoilCallback( ({ set, reset, snapshot: { getPromise } }) => () => { reset(minimizerIndexAtom) reset(autodetectResultsAtom) - void router.push('/autodetect', '/autodetect') // eslint-disable-line no-void - function onResult(res: MinimizerSearchRecord) { - set(autodetectResultAtom(res.fastaRecord.index), res) + set(autodetectResultByIndexAtom(res.fastaRecord.index), res) } Promise.all([getPromise(qrySeqInputsStorageAtom), getPromise(minimizerIndexVersionAtom)]) @@ -40,7 +35,7 @@ export function useRunSeqAutodetect() { throw error }) }, - [router], + [], ) } @@ -93,7 +88,7 @@ export class SeqAutodetectWasmWorker { } async destroy() { - await this.subscription?.unsubscribe() // eslint-disable-line @typescript-eslint/await-thenable + this.subscription?.unsubscribe() await this.thread.destroy() } } diff --git a/packages_rs/nextclade-web/src/hooks/useToggle.ts b/packages_rs/nextclade-web/src/hooks/useToggle.ts index 1b51e6038..da562079c 100644 --- a/packages_rs/nextclade-web/src/hooks/useToggle.ts +++ b/packages_rs/nextclade-web/src/hooks/useToggle.ts @@ -1,4 +1,5 @@ import { useCallback, useState } from 'react' +import { RecoilState, useRecoilState } from 'recoil' export type VoidFunc = () => void @@ -9,3 +10,11 @@ export function useToggle(initialState = false): [boolean, VoidFunc, VoidFunc, V const disable = useCallback(() => setState(false), []) return [state, toggle, enable, disable] } + +export function useRecoilToggle(recoilState: RecoilState) { + const [state, setState] = useRecoilState(recoilState) + const toggle = useCallback(() => setState((state) => !state), [setState]) + const enable = useCallback(() => setState(true), [setState]) + const disable = useCallback(() => setState(false), [setState]) + return { state, setState, toggle, enable, disable } +} diff --git a/packages_rs/nextclade-web/src/pages/autodetect.tsx b/packages_rs/nextclade-web/src/pages/autodetect.tsx deleted file mode 100644 index 3e1888fdb..000000000 --- a/packages_rs/nextclade-web/src/pages/autodetect.tsx +++ /dev/null @@ -1 +0,0 @@ -export { AutodetectPage as default } from 'src/components/Autodetect/AutodetectPage' diff --git a/packages_rs/nextclade-web/src/state/autodetect.state.ts b/packages_rs/nextclade-web/src/state/autodetect.state.ts index 104765f6b..2f793a3e4 100644 --- a/packages_rs/nextclade-web/src/state/autodetect.state.ts +++ b/packages_rs/nextclade-web/src/state/autodetect.state.ts @@ -1,3 +1,4 @@ +import { isNil } from 'lodash' import { atom, atomFamily, DefaultValue, selector, selectorFamily } from 'recoil' import type { MinimizerIndexJson, MinimizerSearchRecord } from 'src/types' import { isDefaultValue } from 'src/state/utils/isDefaultValue' @@ -15,8 +16,8 @@ export const autodetectResultIndicesAtom = atom({ default: [], }) -export const autodetectResultAtom = selectorFamily({ - key: 'autodetectResultAtom', +export const autodetectResultByIndexAtom = selectorFamily({ + key: 'autodetectResultByIndexAtom', get: (index: number) => @@ -44,25 +45,66 @@ export const autodetectResultAtom = selectorFamily({ +// Dataset ID to use for when dataset is not autodetected +export const DATASET_ID_UNDETECTED = 'undetected' + +// Select autodetect results by dataset name +export const autodetectResultsByDatasetAtom = selectorFamily({ + key: 'autodetectResultByDatasetAtom', + + get: + (datasetId: string) => + ({ get }): MinimizerSearchRecord[] | undefined => { + const results = get(autodetectResultsAtom) + if (isNil(results)) { + return undefined + } + + return results.filter((result) => { + if (datasetId === DATASET_ID_UNDETECTED) { + return isNil(result.result.dataset) + } + return result.result.dataset === datasetId + }) + }, +}) + +export const autodetectResultsAtom = selector({ key: 'autodetectResultsAtom', - get({ get }): MinimizerSearchRecord[] { + get({ get }): MinimizerSearchRecord[] | undefined { const indices = get(autodetectResultIndicesAtom) - return indices.map((index) => get(autodetectResultAtom(index))) + if (indices.length === 0) { + return undefined + } + return indices.map((index) => get(autodetectResultByIndexAtom(index))) }, - set({ get, set, reset }, results: MinimizerSearchRecord[] | DefaultValue) { + set({ get, set, reset }, results: MinimizerSearchRecord[] | DefaultValue | undefined) { const seqIndices = get(autodetectResultIndicesAtom) // Remove all results seqIndices.forEach((index) => { - reset(autodetectResultAtom(index)) + reset(autodetectResultByIndexAtom(index)) }) // If the operation is not 'reset', add the new items - if (!isDefaultValue(results)) { - results.forEach((result) => set(autodetectResultAtom(result.fastaRecord.index), result)) + if (!isDefaultValue(results) && !isNil(results)) { + results.forEach((result) => set(autodetectResultByIndexAtom(result.fastaRecord.index), result)) } }, }) + +export const numberAutodetectResultsAtom = selector({ + key: 'numberAutodetectResultsAtom', + get({ get }) { + return (get(autodetectResultsAtom) ?? []).length + }, +}) + +export const hasAutodetectResultsAtom = selector({ + key: 'hasAutodetectResultsAtom', + get({ get }) { + return get(numberAutodetectResultsAtom) > 0 + }, +}) diff --git a/packages_rs/nextclade-web/src/state/settings.state.ts b/packages_rs/nextclade-web/src/state/settings.state.ts index d84aed43e..91f17aa36 100644 --- a/packages_rs/nextclade-web/src/state/settings.state.ts +++ b/packages_rs/nextclade-web/src/state/settings.state.ts @@ -32,7 +32,7 @@ export const isNewRunPopupShownAtom = atom({ }) export const isResultsFilterPanelCollapsedAtom = atom({ - key: 'isResultsfilterPanelCollapsed', + key: 'isResultsFilterPanelCollapsedAtom', default: true, }) @@ -42,6 +42,12 @@ export const shouldRunAutomaticallyAtom = atom({ effects: [persistAtom], }) +export const shouldSuggestDatasetsAtom = atom({ + key: 'shouldSuggestDatasetsAtom', + default: true, + effects: [persistAtom], +}) + export const changelogIsShownAtom = atom({ key: 'changelogIsShown', default: false, From 638da5704150e20154e117bd185305683ce8c728 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Thu, 7 Sep 2023 10:28:52 +0200 Subject: [PATCH 20/51] feat(web): animate list updates to prevent flickering --- packages_rs/nextclade-web/package.json | 1 + .../components/Main/DatasetSelectorList.tsx | 32 ++++++++++++++----- packages_rs/nextclade-web/yarn.lock | 26 +++++++++++++++ 3 files changed, 51 insertions(+), 8 deletions(-) diff --git a/packages_rs/nextclade-web/package.json b/packages_rs/nextclade-web/package.json index 0c7446ac1..bf0a10db6 100644 --- a/packages_rs/nextclade-web/package.json +++ b/packages_rs/nextclade-web/package.json @@ -105,6 +105,7 @@ "file-saver": "2.0.5", "flag-icon-css": "3.5.0", "formik": "2.2.9", + "framer-motion": "10.16.4", "history": "5.3.0", "i18next": "19.3.2", "immutable": "4.0.0", diff --git a/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx b/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx index dc82ac797..1b7330254 100644 --- a/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx +++ b/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx @@ -1,8 +1,8 @@ import { isNil, sortBy } from 'lodash' -import React, { useCallback, useMemo } from 'react' -import { ListGroup, ListGroupItem } from 'reactstrap' +import React, { HTMLProps, useCallback, useMemo } from 'react' import { useRecoilValue } from 'recoil' import styled from 'styled-components' +import { motion } from 'framer-motion' import type { Dataset } from 'src/types' import { areDatasetsEqual } from 'src/types' import { autodetectResultsAtom } from 'src/state/autodetect.state' @@ -10,21 +10,30 @@ import { autodetectResultsAtom } from 'src/state/autodetect.state' // import { search } from 'src/helpers/search' import { DatasetInfo } from 'src/components/Main/DatasetInfo' -export const DatasetSelectorUl = styled(ListGroup)` +export const DatasetSelectorUl = styled.ul` flex: 1; overflow-y: scroll; height: 100%; + padding: 0; ` -export const DatasetSelectorLi = styled(ListGroupItem)<{ $isDimmed?: boolean }>` +export const DatasetSelectorLi = styled(motion.li)<{ $active?: boolean; $isDimmed?: boolean }>` list-style: none; margin: 0; padding: 0.5rem; cursor: pointer; - opacity: ${(props) => props.$isDimmed && 0.33}; - background-color: transparent; + filter: ${(props) => props.$isDimmed && !props.$active && 'invert(0.1) brightness(0.9)'}; + background-color: ${(props) => (props.$active ? props.theme.primary : props.theme.bodyBg)}; + color: ${(props) => props.$active && props.theme.white}; + border: ${(props) => props.theme.gray400} solid 1px; ` +const TRANSITION = { + type: 'tween', + ease: 'linear', + duration: 0.2, +} + export interface DatasetSelectorListItemProps { dataset: Dataset isCurrent?: boolean @@ -34,13 +43,20 @@ export interface DatasetSelectorListItemProps { export function DatasetSelectorListItem({ dataset, isCurrent, isDimmed, onClick }: DatasetSelectorListItemProps) { return ( - + ) } -export interface DatasetSelectorListProps { +export interface DatasetSelectorListProps extends HTMLProps { datasets: Dataset[] searchTerm: string datasetHighlighted?: Dataset diff --git a/packages_rs/nextclade-web/yarn.lock b/packages_rs/nextclade-web/yarn.lock index 5c15055fe..71cf0a43f 100644 --- a/packages_rs/nextclade-web/yarn.lock +++ b/packages_rs/nextclade-web/yarn.lock @@ -1998,6 +1998,13 @@ resolved "https://registry.yarnpkg.com/@emotion/hash/-/hash-0.8.0.tgz#bbbff68978fefdbe68ccb533bc8cbe1d1afb5413" integrity sha512-kBJtf7PH6aWwZ6fka3zQ0p6SBYzx4fl1LoZXE2RrnYST9Xljm7WfKJrU4g/Xr3Beg72MLrp1AWNUmuYJTL7Cow== +"@emotion/is-prop-valid@^0.8.2": + version "0.8.8" + resolved "https://registry.yarnpkg.com/@emotion/is-prop-valid/-/is-prop-valid-0.8.8.tgz#db28b1c4368a259b60a97311d6a952d4fd01ac1a" + integrity sha512-u5WtneEAr5IDG2Wv65yhunPSMLIpuKsbuOktRojfrEiEvRyC85LgPMZI63cr7NUqT8ZIGdSVg8ZKGxIug4lXcA== + dependencies: + "@emotion/memoize" "0.7.4" + "@emotion/is-prop-valid@^1.1.0": version "1.1.2" resolved "https://registry.yarnpkg.com/@emotion/is-prop-valid/-/is-prop-valid-1.1.2.tgz#34ad6e98e871aa6f7a20469b602911b8b11b3a95" @@ -2005,6 +2012,11 @@ dependencies: "@emotion/memoize" "^0.7.4" +"@emotion/memoize@0.7.4": + version "0.7.4" + resolved "https://registry.yarnpkg.com/@emotion/memoize/-/memoize-0.7.4.tgz#19bf0f5af19149111c40d98bb0cf82119f5d9eeb" + integrity sha512-Ja/Vfqe3HpuzRsG1oBtWTHk2PGZ7GR+2Vz5iYGelAw8dx32K0y7PjVuxK6z1nMpZOqAFsRUPCkK1YjJ56qJlgw== + "@emotion/memoize@^0.7.4", "@emotion/memoize@^0.7.5": version "0.7.5" resolved "https://registry.yarnpkg.com/@emotion/memoize/-/memoize-0.7.5.tgz#2c40f81449a4e554e9fc6396910ed4843ec2be50" @@ -8009,6 +8021,15 @@ fragment-cache@^0.2.1: dependencies: map-cache "^0.2.2" +framer-motion@10.16.4: + version "10.16.4" + resolved "https://registry.yarnpkg.com/framer-motion/-/framer-motion-10.16.4.tgz#30279ef5499b8d85db3a298ee25c83429933e9f8" + integrity sha512-p9V9nGomS3m6/CALXqv6nFGMuFOxbWsmaOrdmhyQimMIlLl3LC7h7l86wge/Js/8cRu5ktutS/zlzgR7eBOtFA== + dependencies: + tslib "^2.4.0" + optionalDependencies: + "@emotion/is-prop-valid" "^0.8.2" + fresh@0.5.2: version "0.5.2" resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" @@ -15803,6 +15824,11 @@ tslib@^2.0.3, tslib@^2.2.0, tslib@^2.3.1: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.4.0.tgz#7cecaa7f073ce680a05847aa77be941098f36dc3" integrity sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ== +tslib@^2.4.0: + version "2.6.2" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" + integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== + tsutils@^3.21.0: version "3.21.0" resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.21.0.tgz#b48717d394cea6c1e096983eed58e9d61715b623" From 94578f41399f939d0df6daaa414e16090265541c Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Thu, 7 Sep 2023 10:40:20 +0200 Subject: [PATCH 21/51] perf: serialize directly to jsvalue to avoid json parsing --- packages_rs/nextclade-web/src/wasm/seq_autodetect.rs | 8 ++------ .../src/workers/nextcladeAutodetect.worker.ts | 9 ++++----- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/packages_rs/nextclade-web/src/wasm/seq_autodetect.rs b/packages_rs/nextclade-web/src/wasm/seq_autodetect.rs index 54cbeab85..ce000bec9 100644 --- a/packages_rs/nextclade-web/src/wasm/seq_autodetect.rs +++ b/packages_rs/nextclade-web/src/wasm/seq_autodetect.rs @@ -1,7 +1,6 @@ use crate::wasm::jserr::jserr; use eyre::WrapErr; use nextclade::io::fasta::{FastaReader, FastaRecord}; -use nextclade::io::json::{json_stringify, JsonPretty}; use nextclade::sort::minimizer_index::MinimizerIndexJson; use nextclade::sort::minimizer_search::{run_minimizer_search, MinimizerSearchRecord}; use nextclade::sort::params::NextcladeSeqSortParams; @@ -44,13 +43,10 @@ impl NextcladeSeqAutodetectWasm { }), )?; - let result_json = jserr(json_stringify( - &MinimizerSearchRecord { fasta_record, result }, - JsonPretty(false), - ))?; + let result_js = serde_wasm_bindgen::to_value(&MinimizerSearchRecord { fasta_record, result })?; callback - .call1(&JsValue::null(), &JsValue::from_str(&result_json)) + .call1(&JsValue::null(), &result_js) .map_err(|err_val| JsError::new(&format!("{err_val:#?}")))?; } diff --git a/packages_rs/nextclade-web/src/workers/nextcladeAutodetect.worker.ts b/packages_rs/nextclade-web/src/workers/nextcladeAutodetect.worker.ts index 383c77f1a..802e022e3 100644 --- a/packages_rs/nextclade-web/src/workers/nextcladeAutodetect.worker.ts +++ b/packages_rs/nextclade-web/src/workers/nextcladeAutodetect.worker.ts @@ -10,6 +10,10 @@ import { NextcladeSeqAutodetectWasm } from 'src/gen/nextclade-wasm' const gSubject = new Subject() +function onResultParsed(resStr: MinimizerSearchRecord) { + gSubject.next(resStr) +} + /** * Keeps the reference to the WebAssembly module.The module is stateful and requires manual initialization * and teardown. @@ -37,11 +41,6 @@ async function autodetect(fasta: string): Promise { throw new ErrorModuleNotInitialized('autodetect') } - function onResultParsed(resStr: string) { - const result = JSON.parse(resStr) as MinimizerSearchRecord - gSubject.next(result) - } - try { nextcladeAutodetect.autodetect(fasta, onResultParsed) } catch (error: unknown) { From 2a975618df9d3613e34cb661e2c8b37e11f01ae5 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Thu, 7 Sep 2023 11:18:17 +0200 Subject: [PATCH 22/51] perf: batch updates from wasm to reduce main thread blocking --- Cargo.lock | 1 + packages_rs/nextclade-web/Cargo.toml | 3 +- .../src/hooks/useRunSeqAutodetect.ts | 12 +++-- .../src/state/autodetect.state.ts | 2 +- .../nextclade-web/src/wasm/seq_autodetect.rs | 50 +++++++++++++++++-- .../src/workers/nextcladeAutodetect.worker.ts | 11 ++-- 6 files changed, 63 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 70a948561..3da8135d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1820,6 +1820,7 @@ name = "nextclade-web" version = "3.0.0-alpha.0" dependencies = [ "assert2", + "chrono", "console_error_panic_hook", "eyre", "getrandom", diff --git a/packages_rs/nextclade-web/Cargo.toml b/packages_rs/nextclade-web/Cargo.toml index 8288ad8f1..08f8b8da2 100644 --- a/packages_rs/nextclade-web/Cargo.toml +++ b/packages_rs/nextclade-web/Cargo.toml @@ -12,6 +12,7 @@ crate-type = ["cdylib", "rlib"] [dependencies] assert2 = "=0.3.11" +chrono = { version = "=0.4.26", default-features = false, features = ["clock", "std", "wasmbind"] } console_error_panic_hook = "=0.1.7" eyre = "=0.6.8" getrandom = { version = "=0.2.10", features = ["js"] } @@ -19,12 +20,12 @@ itertools = "=0.11.0" js-sys = { version = "=0.3.64", features = [] } log = "=0.4.19" nextclade = { path = "../nextclade" } +schemars = { version = "=0.8.12", features = ["chrono", "either", "enumset", "indexmap1"] } serde = { version = "=1.0.164", features = ["derive"] } serde-wasm-bindgen = { version = "=0.5.0" } wasm-bindgen = { version = "=0.2.87", features = ["serde-serialize"] } wasm-logger = "=0.2.0" web-sys = { version = "=0.3.64", features = ["console"] } -schemars = { version = "=0.8.12", features = ["chrono", "either", "enumset", "indexmap1"] } [build-dependencies] nextclade = { path = "../nextclade" } diff --git a/packages_rs/nextclade-web/src/hooks/useRunSeqAutodetect.ts b/packages_rs/nextclade-web/src/hooks/useRunSeqAutodetect.ts index 80ae6590c..ce0af4768 100644 --- a/packages_rs/nextclade-web/src/hooks/useRunSeqAutodetect.ts +++ b/packages_rs/nextclade-web/src/hooks/useRunSeqAutodetect.ts @@ -17,8 +17,10 @@ export function useRunSeqAutodetect() { reset(minimizerIndexAtom) reset(autodetectResultsAtom) - function onResult(res: MinimizerSearchRecord) { - set(autodetectResultByIndexAtom(res.fastaRecord.index), res) + function onResult(results: MinimizerSearchRecord[]) { + results.forEach((res) => { + set(autodetectResultByIndexAtom(res.fastaRecord.index), res) + }) } Promise.all([getPromise(qrySeqInputsStorageAtom), getPromise(minimizerIndexVersionAtom)]) @@ -42,7 +44,7 @@ export function useRunSeqAutodetect() { async function runAutodetect( fasta: string, minimizerIndex: MinimizerIndexJson, - onResult: (res: MinimizerSearchRecord) => void, + onResult: (res: MinimizerSearchRecord[]) => void, ) { const worker = await SeqAutodetectWasmWorker.create(minimizerIndex) await worker.autodetect(fasta, { onResult }) @@ -51,7 +53,7 @@ async function runAutodetect( export class SeqAutodetectWasmWorker { private thread!: NextcladeSeqAutodetectWasmWorker - private subscription?: Subscription + private subscription?: Subscription private constructor() {} @@ -78,7 +80,7 @@ export class SeqAutodetectWasmWorker { onError, onComplete, }: { - onResult: (r: MinimizerSearchRecord) => void + onResult: (r: MinimizerSearchRecord[]) => void onError?: (error: Error) => void onComplete?: () => void }, diff --git a/packages_rs/nextclade-web/src/state/autodetect.state.ts b/packages_rs/nextclade-web/src/state/autodetect.state.ts index 2f793a3e4..11913819c 100644 --- a/packages_rs/nextclade-web/src/state/autodetect.state.ts +++ b/packages_rs/nextclade-web/src/state/autodetect.state.ts @@ -36,7 +36,7 @@ export const autodetectResultByIndexAtom = selectorFamily { - if (result && !prev.includes(result.fastaRecord.index)) { + if (result) { return [...prev, result.fastaRecord.index] } return prev diff --git a/packages_rs/nextclade-web/src/wasm/seq_autodetect.rs b/packages_rs/nextclade-web/src/wasm/seq_autodetect.rs index ce000bec9..5219ba526 100644 --- a/packages_rs/nextclade-web/src/wasm/seq_autodetect.rs +++ b/packages_rs/nextclade-web/src/wasm/seq_autodetect.rs @@ -1,32 +1,59 @@ use crate::wasm::jserr::jserr; +use chrono::Duration; use eyre::WrapErr; use nextclade::io::fasta::{FastaReader, FastaRecord}; +use nextclade::io::json::json_parse; use nextclade::sort::minimizer_index::MinimizerIndexJson; use nextclade::sort::minimizer_search::{run_minimizer_search, MinimizerSearchRecord}; use nextclade::sort::params::NextcladeSeqSortParams; +use nextclade::utils::datetime::date_now; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; use std::io::Read; use std::str::FromStr; use wasm_bindgen::prelude::*; +#[wasm_bindgen] +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct NextcladeSeqAutodetectWasmParams { + batch_interval_ms: i64, + max_batch_size: usize, +} + +impl Default for NextcladeSeqAutodetectWasmParams { + fn default() -> Self { + Self { + batch_interval_ms: 500, + max_batch_size: 100, + } + } +} + #[wasm_bindgen] pub struct NextcladeSeqAutodetectWasm { minimizer_index: MinimizerIndexJson, search_params: NextcladeSeqSortParams, + run_params: NextcladeSeqAutodetectWasmParams, } #[wasm_bindgen] impl NextcladeSeqAutodetectWasm { - pub fn new(minimizer_index_json_str: &str) -> Result { + pub fn new(minimizer_index_json_str: &str, params: &str) -> Result { let minimizer_index = jserr(MinimizerIndexJson::from_str(minimizer_index_json_str))?; Ok(Self { minimizer_index, search_params: NextcladeSeqSortParams::default(), + run_params: jserr(json_parse(params))?, }) } pub fn autodetect(&self, qry_fasta_str: &str, callback: &js_sys::Function) -> Result<(), JsError> { let mut reader = jserr(FastaReader::from_str(&qry_fasta_str).wrap_err_with(|| "When creating fasta reader"))?; + let mut batch = vec![]; + let mut last_flush = date_now(); + loop { let mut fasta_record = FastaRecord::default(); jserr(reader.read(&mut fasta_record).wrap_err("When reading a fasta record"))?; @@ -43,13 +70,26 @@ impl NextcladeSeqAutodetectWasm { }), )?; - let result_js = serde_wasm_bindgen::to_value(&MinimizerSearchRecord { fasta_record, result })?; + batch.push(MinimizerSearchRecord { fasta_record, result }); - callback - .call1(&JsValue::null(), &result_js) - .map_err(|err_val| JsError::new(&format!("{err_val:#?}")))?; + if (date_now() - last_flush >= Duration::milliseconds(self.run_params.batch_interval_ms)) + || batch.len() >= self.run_params.max_batch_size + { + let result_js = serde_wasm_bindgen::to_value(&batch)?; + callback + .call1(&JsValue::null(), &result_js) + .map_err(|err_val| JsError::new(&format!("{err_val:#?}")))?; + last_flush = date_now(); + batch.clear(); + } } + let result_js = serde_wasm_bindgen::to_value(&batch)?; + callback + .call1(&JsValue::null(), &result_js) + .map_err(|err_val| JsError::new(&format!("{err_val:#?}")))?; + batch.clear(); + Ok(()) } } diff --git a/packages_rs/nextclade-web/src/workers/nextcladeAutodetect.worker.ts b/packages_rs/nextclade-web/src/workers/nextcladeAutodetect.worker.ts index 802e022e3..3343331b7 100644 --- a/packages_rs/nextclade-web/src/workers/nextcladeAutodetect.worker.ts +++ b/packages_rs/nextclade-web/src/workers/nextcladeAutodetect.worker.ts @@ -8,9 +8,9 @@ import type { Thread } from 'threads' import { expose } from 'threads/worker' import { NextcladeSeqAutodetectWasm } from 'src/gen/nextclade-wasm' -const gSubject = new Subject() +const gSubject = new Subject() -function onResultParsed(resStr: MinimizerSearchRecord) { +function onResultParsed(resStr: MinimizerSearchRecord[]) { gSubject.next(resStr) } @@ -23,7 +23,10 @@ let nextcladeAutodetect: NextcladeSeqAutodetectWasm | undefined /** Creates the underlying WebAssembly module. */ async function create(minimizerIndexJsonStr: MinimizerIndexJson) { - nextcladeAutodetect = NextcladeSeqAutodetectWasm.new(JSON.stringify(minimizerIndexJsonStr)) + nextcladeAutodetect = NextcladeSeqAutodetectWasm.new( + JSON.stringify(minimizerIndexJsonStr), + JSON.stringify({ batchIntervalMs: 700, maxBatchSize: 1000 }), + ) } /** Destroys the underlying WebAssembly module. */ @@ -54,7 +57,7 @@ const worker = { create, destroy, autodetect, - values(): ThreadsObservable { + values(): ThreadsObservable { return ThreadsObservable.from(gSubject) }, } From c52e8d813540e78bf53b785f734fc011dd890771 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Thu, 7 Sep 2023 11:45:30 +0200 Subject: [PATCH 23/51] fix(web): duplicate id and label misalignment --- packages_rs/nextclade-web/src/components/Common/Toggle.tsx | 2 ++ .../nextclade-web/src/components/Main/DatasetSelector.tsx | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages_rs/nextclade-web/src/components/Common/Toggle.tsx b/packages_rs/nextclade-web/src/components/Common/Toggle.tsx index ab6afbcb6..5f15d029a 100644 --- a/packages_rs/nextclade-web/src/components/Common/Toggle.tsx +++ b/packages_rs/nextclade-web/src/components/Common/Toggle.tsx @@ -6,6 +6,8 @@ import ReactToggle, { ToggleProps as ReactToggleProps } from 'react-toggle' import 'react-toggle/style.css' export const ToggleBase = styled(ReactToggle)` + display: block; + &.react-toggle-custom { & > .react-toggle-track { background-color: #9c3434; diff --git a/packages_rs/nextclade-web/src/components/Main/DatasetSelector.tsx b/packages_rs/nextclade-web/src/components/Main/DatasetSelector.tsx index 7dcc92cc3..2fb2eae9b 100644 --- a/packages_rs/nextclade-web/src/components/Main/DatasetSelector.tsx +++ b/packages_rs/nextclade-web/src/components/Main/DatasetSelector.tsx @@ -135,7 +135,7 @@ function AutodetectToggle() {
From 7c2a3050b88eabddef7274e40f29c381b6294280 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Thu, 7 Sep 2023 22:38:20 +0200 Subject: [PATCH 24/51] feat: count detections for all datasets, not just the top one Also precompute and presort detections in a more convenient way --- Cargo.lock | 10 +- .../src/cli/nextclade_seq_sort.rs | 25 +- .../components/Autodetect/AutodetectPage.tsx | 508 +++++++++--------- .../src/components/Main/DatasetInfo.tsx | 17 +- .../components/Main/DatasetSelectorList.tsx | 49 +- .../src/state/autodetect.state.ts | 53 +- .../nextclade-web/src/wasm/seq_autodetect.rs | 10 +- .../src/workers/nextcladeAutodetect.worker.ts | 4 +- packages_rs/nextclade/Cargo.toml | 1 + .../nextclade/src/sort/minimizer_search.rs | 65 +-- packages_rs/nextclade/src/sort/params.rs | 16 +- 11 files changed, 405 insertions(+), 353 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3da8135d1..86da874b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1748,6 +1748,7 @@ dependencies = [ "num-traits", "num_cpus", "optfield", + "ordered-float", "owo-colors", "pretty_assertions", "rayon", @@ -1958,11 +1959,14 @@ dependencies = [ [[package]] name = "ordered-float" -version = "3.7.0" +version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fc2dbde8f8a79f2102cc474ceb0ad68e3b80b85289ea62389b60e66777e4213" +checksum = "2a54938017eacd63036332b4ae5c8a49fc8c0c1d6d629893057e4f13609edd06" dependencies = [ "num-traits", + "rand", + "schemars", + "serde", ] [[package]] @@ -2178,6 +2182,7 @@ dependencies = [ "libc", "rand_chacha", "rand_core", + "serde", ] [[package]] @@ -2197,6 +2202,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ "getrandom", + "serde", ] [[package]] diff --git a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs index 670537bf5..8e071f018 100644 --- a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs +++ b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs @@ -103,7 +103,7 @@ pub fn run(args: &NextcladeSeqSortArgs, minimizer_index: &MinimizerIndexJson) -> for fasta_record in &fasta_receiver { info!("Processing sequence '{}'", fasta_record.seq_name); - let result = run_minimizer_search(&fasta_record, minimizer_index, search_params) + let result = run_minimizer_search(&fasta_record, minimizer_index) .wrap_err_with(|| { format!( "When processing sequence #{} '{}'", @@ -112,10 +112,14 @@ pub fn run(args: &NextcladeSeqSortArgs, minimizer_index: &MinimizerIndexJson) -> }) .unwrap(); - result_sender - .send(MinimizerSearchRecord { fasta_record, result }) - .wrap_err("When sending minimizer record into the channel") - .unwrap(); + if result.max_score >= search_params.min_score + && result.total_hits >= search_params.min_hits + { + result_sender + .send(MinimizerSearchRecord { fasta_record, result }) + .wrap_err("When sending minimizer record into the channel") + .unwrap(); + } } drop(result_sender); @@ -142,7 +146,11 @@ pub fn run(args: &NextcladeSeqSortArgs, minimizer_index: &MinimizerIndexJson) -> let mut writers = BTreeMap::new(); for record in result_receiver { - if let Some(name) = &record.result.dataset { + let dataset = record.result.datasets.first(); + + if let Some(dataset) = dataset { + let name = &dataset.name; + let filepath = match (&tt, output_dir) { (Some(tt), None) => { let filepath_str = tt @@ -172,12 +180,13 @@ pub fn run(args: &NextcladeSeqSortArgs, minimizer_index: &MinimizerIndexJson) -> } } + let name_or_empty = dataset.as_ref().map(|dataset| dataset.name.clone()).unwrap_or_default(); println!( "{:40} | {:40} | {:>10} | {:>.3}", &truncate(record.fasta_record.seq_name, 40), - &truncate(record.result.dataset.unwrap_or_default(), 40), + &truncate(name_or_empty, 40), &record.result.total_hits, - &record.result.max_normalized_hit + &record.result.max_score ); } }); diff --git a/packages_rs/nextclade-web/src/components/Autodetect/AutodetectPage.tsx b/packages_rs/nextclade-web/src/components/Autodetect/AutodetectPage.tsx index d305e0a8d..4dd2a51e6 100644 --- a/packages_rs/nextclade-web/src/components/Autodetect/AutodetectPage.tsx +++ b/packages_rs/nextclade-web/src/components/Autodetect/AutodetectPage.tsx @@ -1,259 +1,249 @@ -import classNames from 'classnames' -import { sortBy } from 'lodash' -import { mix, transparentize } from 'polished' -import React, { useMemo } from 'react' -import { Col as ColBase, Row as RowBase } from 'reactstrap' -import { useRecoilValue } from 'recoil' -import styled, { useTheme } from 'styled-components' -import type { MinimizerIndexJson, MinimizerSearchRecord } from 'src/types' -import { isEven } from 'src/helpers/number' -import { TableSlim } from 'src/components/Common/TableSlim' -import { Layout } from 'src/components/Layout/Layout' -import { safeZip3 } from 'src/helpers/safeZip' -import { useTranslationSafe } from 'src/helpers/useTranslationSafe' -import { autodetectResultsAtom, minimizerIndexAtom } from 'src/state/autodetect.state' - -const Container = styled.div` - margin-top: 1rem; - padding-bottom: 1.5rem; - height: 100%; - overflow: hidden; -` - -const Row = styled(RowBase)` - overflow: hidden; - height: 100%; -` - -const Col = styled(ColBase)` - overflow: hidden; - height: 100%; -` - -const Table = styled(TableSlim)` - padding-top: 50px; - - & thead { - height: 51px; - position: sticky; - top: -2px; - background-color: ${(props) => props.theme.gray700}; - color: ${(props) => props.theme.gray100}; - } - - & thead th { - margin: auto; - text-align: center; - vertical-align: middle; - } - - & td { - border: none; - border-left: 1px solid #ccc; - } - - & tr { - border: none !important; - } - - & th { - border: 1px solid #ccc; - } -` - -const TableWrapper = styled.div` - height: 100%; - overflow-y: auto; -` - -export function AutodetectPage() { - const { t } = useTranslationSafe() - const minimizerIndex = useRecoilValue(minimizerIndexAtom) - const autodetectResults = useRecoilValue(autodetectResultsAtom) - - const rows = useMemo(() => { - const results = sortBy(autodetectResults, (result) => result.fastaRecord.index) - return results.map((res, i) => ( - - )) - }, [autodetectResults, minimizerIndex]) - - return ( - - - - - - - - - - - - - - - - - - - - - {rows} -
{'#'}{t('Seq. name')}{t('Length')}{t('Total hits')}{t('Max norm. hit')}{t('Dataset')}{t('Ref. length')}{t('Num. hits')}{t('Norm. hit')}
- - - - - - ) -} - -interface AutodetectTableRowSpanProps { - order: number - res: MinimizerSearchRecord - minimizerIndex: MinimizerIndexJson -} - -function AutodetectTableRowSpan({ order, res, minimizerIndex }: AutodetectTableRowSpanProps) { - const theme = useTheme() - - const { hitCounts, maxNormalizedHit, normalizedHits, totalHits } = res.result - const { seqName, index: seqIndex, seq } = res.fastaRecord - const qryLen = seq.length - - const rows = useMemo(() => { - let entries = safeZip3(normalizedHits, hitCounts, minimizerIndex.references ?? []).map(([score, hits, ref]) => { - return { dataset: ref.name, refLen: ref.length, score, hits } - }) - - entries = sortBy(entries, (entry) => -entry.score) - - let color = isEven(order) ? theme.table.rowBg.even : theme.table.rowBg.odd - - const goodEntries = entries.filter( - ({ score, hits }) => maxNormalizedHit >= 0.6 && hits >= 10 && score >= maxNormalizedHit * 0.5, - ) - - const mediocreEntries = entries.filter( - ({ score, hits }) => maxNormalizedHit >= 0.3 && hits >= 10 && score >= maxNormalizedHit * 0.5, - ) - - const badEntries = entries.filter( - ({ score, hits }) => maxNormalizedHit >= 0.05 && hits > 0 && score >= maxNormalizedHit * 0.5, - ) - - if (goodEntries.length > 0) { - entries = goodEntries - } else if (mediocreEntries.length > 0) { - entries = mediocreEntries - color = mix(0.3, transparentize(0.3)(theme.warning), color) - } else { - entries = badEntries - color = mix(0.5, transparentize(0.5)(theme.danger), color) - } - - return entries.map(({ dataset, score, hits, refLen }, i) => { - const cls = classNames(i === 0 && 'font-weight-bold') - - return ( - - {i === 0 && ( - <> - - {seqIndex} - - - - {seqName} - - - - {qryLen} - - - - {totalHits} - - - - {maxNormalizedHit.toFixed(3)} - - - )} - - {dataset} - - - {refLen} - - - {hits} - - - {score.toFixed(3)} - - - ) - }) - }, [ - normalizedHits, - hitCounts, - minimizerIndex.references, - order, - theme.table.rowBg.even, - theme.table.rowBg.odd, - theme.warning, - theme.danger, - maxNormalizedHit, - seqIndex, - seqName, - qryLen, - totalHits, - ]) - - return ( - <> - {rows} - - - - - ) -} - -const Tr = styled.tr<{ $bg?: string }>` - background-color: ${(props) => props.$bg}; -` - -const Td = styled.td` - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; - max-width: 100px; - font-size: 0.95rem; -` - -const TdName = styled(Td)` - min-width: 200px; - font-size: 0.9rem; -` - -const TdNumeric = styled(Td)` - text-align: right; - font-family: ${(props) => props.theme.font.monospace}; - font-size: 0.9rem; -` - -const TdIndex = styled(TdNumeric)` - background-color: ${(props) => props.theme.gray700}; - color: ${(props) => props.theme.gray100}; -` - -const TrSpacer = styled.tr` - height: 2px; - - & td { - background-color: ${(props) => props.theme.gray400}; - } -` +// import classNames from 'classnames' +// import { sortBy } from 'lodash' +// import { mix, transparentize } from 'polished' +// import React, { useMemo } from 'react' +// import { Col as ColBase, Row as RowBase } from 'reactstrap' +// import { useRecoilValue } from 'recoil' +// import styled, { useTheme } from 'styled-components' +// import type { MinimizerIndexJson, MinimizerSearchRecord } from 'src/types' +// import { isEven } from 'src/helpers/number' +// import { TableSlim } from 'src/components/Common/TableSlim' +// import { Layout } from 'src/components/Layout/Layout' +// import { safeZip3 } from 'src/helpers/safeZip' +// import { useTranslationSafe } from 'src/helpers/useTranslationSafe' +// import { autodetectResultsAtom, minimizerIndexAtom } from 'src/state/autodetect.state' +// +// const Container = styled.div` +// margin-top: 1rem; +// padding-bottom: 1.5rem; +// height: 100%; +// overflow: hidden; +// ` +// +// const Row = styled(RowBase)` +// overflow: hidden; +// height: 100%; +// ` +// +// const Col = styled(ColBase)` +// overflow: hidden; +// height: 100%; +// ` +// +// const Table = styled(TableSlim)` +// padding-top: 50px; +// +// & thead { +// height: 51px; +// position: sticky; +// top: -2px; +// background-color: ${(props) => props.theme.gray700}; +// color: ${(props) => props.theme.gray100}; +// } +// +// & thead th { +// margin: auto; +// text-align: center; +// vertical-align: middle; +// } +// +// & td { +// border: none; +// border-left: 1px solid #ccc; +// } +// +// & tr { +// border: none !important; +// } +// +// & th { +// border: 1px solid #ccc; +// } +// ` +// +// const TableWrapper = styled.div` +// height: 100%; +// overflow-y: auto; +// ` +// +// export function AutodetectPage() { +// const { t } = useTranslationSafe() +// const minimizerIndex = useRecoilValue(minimizerIndexAtom) +// const autodetectResults = useRecoilValue(autodetectResultsAtom) +// +// const rows = useMemo(() => { +// const results = sortBy(autodetectResults, (result) => result.fastaRecord.index) +// return results.map((res, i) => ( +// +// )) +// }, [autodetectResults, minimizerIndex]) +// +// return ( +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// {rows} +//
{'#'}{t('Seq. name')}{t('Length')}{t('Total nHits')}{t('Max norm. hit')}{t('Dataset')}{t('Ref. length')}{t('Num. nHits')}{t('Norm. hit')}
+// +// +// +// +// +// ) +// } +// +// interface AutodetectTableRowSpanProps { +// order: number +// res: MinimizerSearchRecord +// minimizerIndex: MinimizerIndexJson +// } +// +// function AutodetectTableRowSpan({ order, res, minimizerIndex }: AutodetectTableRowSpanProps) { +// const theme = useTheme() +// +// const { datasets, maxScore, totalHits } = res.result +// const { seqName, index: seqIndex, seq } = res.fastaRecord +// const qryLen = seq.length +// +// const rows = useMemo(() => { +// let entries = sortBy(datasets, (entry) => -entry.score) +// +// let color = isEven(order) ? theme.table.rowBg.even : theme.table.rowBg.odd +// +// const goodEntries = entries.filter(({ score, nHits }) => maxScore >= 0.6 && nHits >= 10 && score >= maxScore * 0.5) +// +// const mediocreEntries = entries.filter( +// ({ score, nHits }) => maxScore >= 0.3 && nHits >= 10 && score >= maxScore * 0.5, +// ) +// +// const badEntries = entries.filter(({ score, nHits }) => maxScore >= 0.05 && nHits > 0 && score >= maxScore * 0.5) +// +// if (goodEntries.length > 0) { +// entries = goodEntries +// } else if (mediocreEntries.length > 0) { +// entries = mediocreEntries +// color = mix(0.3, transparentize(0.3)(theme.warning), color) +// } else { +// entries = badEntries +// color = mix(0.5, transparentize(0.5)(theme.danger), color) +// } +// +// return entries.map(({ dataset, score, nHits, refLen }, i) => { +// const cls = classNames(i === 0 && 'font-weight-bold') +// +// return ( +// +// {i === 0 && ( +// <> +// +// {seqIndex} +// +// +// +// {seqName} +// +// +// +// {qryLen} +// +// +// +// {totalHits} +// +// +// +// {maxScore.toFixed(3)} +// +// +// )} +// +// {dataset} +// +// +// {refLen} +// +// +// {nHits} +// +// +// {score.toFixed(3)} +// +// +// ) +// }) +// }, [ +// datasets, +// order, +// theme.table.rowBg.even, +// theme.table.rowBg.odd, +// theme.warning, +// theme.danger, +// maxScore, +// seqIndex, +// seqName, +// qryLen, +// totalHits, +// ]) +// +// return ( +// <> +// {rows} +// +// +// +// +// ) +// } +// +// const Tr = styled.tr<{ $bg?: string }>` +// background-color: ${(props) => props.$bg}; +// ` +// +// const Td = styled.td` +// white-space: nowrap; +// overflow: hidden; +// text-overflow: ellipsis; +// max-width: 100px; +// font-size: 0.95rem; +// ` +// +// const TdName = styled(Td)` +// min-width: 200px; +// font-size: 0.9rem; +// ` +// +// const TdNumeric = styled(Td)` +// text-align: right; +// font-family: ${(props) => props.theme.font.monospace}; +// font-size: 0.9rem; +// ` +// +// const TdIndex = styled(TdNumeric)` +// background-color: ${(props) => props.theme.gray700}; +// color: ${(props) => props.theme.gray100}; +// ` +// +// const TrSpacer = styled.tr` +// height: 2px; +// +// & td { +// background-color: ${(props) => props.theme.gray400}; +// } +// ` diff --git a/packages_rs/nextclade-web/src/components/Main/DatasetInfo.tsx b/packages_rs/nextclade-web/src/components/Main/DatasetInfo.tsx index 39a435a73..1d8b97871 100644 --- a/packages_rs/nextclade-web/src/components/Main/DatasetInfo.tsx +++ b/packages_rs/nextclade-web/src/components/Main/DatasetInfo.tsx @@ -10,6 +10,7 @@ import { useTranslationSafe } from 'src/helpers/useTranslationSafe' import { autodetectResultsByDatasetAtom, DATASET_ID_UNDETECTED, + filterGoodRecords, numberAutodetectResultsAtom, } from 'src/state/autodetect.state' import type { Dataset } from 'src/types' @@ -172,11 +173,11 @@ function DatasetInfoAutodetectProgressCircle({ dataset }: DatasetInfoCircleProps const { name } = attributes const circleBg = useMemo(() => darken(0.1)(colorHash(path, { saturation: 0.5, reverse: true })), [path]) - const autodetectResults = useRecoilValue(autodetectResultsByDatasetAtom(path)) + const records = useRecoilValue(autodetectResultsByDatasetAtom(path)) const numberAutodetectResults = useRecoilValue(numberAutodetectResultsAtom) const { circleText, countText, percentage } = useMemo(() => { - if (isNil(autodetectResults)) { + if (isNil(records)) { return { circleText: (firstLetter(name.valueFriendly ?? name.value) ?? ' ').toUpperCase(), percentage: 0, @@ -184,14 +185,16 @@ function DatasetInfoAutodetectProgressCircle({ dataset }: DatasetInfoCircleProps } } - if (autodetectResults.length > 0) { - const percentage = autodetectResults.length / numberAutodetectResults + const goodRecords = filterGoodRecords(records) + + if (goodRecords.length > 0) { + const percentage = goodRecords.length / numberAutodetectResults const circleText = `${(100 * percentage).toFixed(0)}%` - const countText = `${autodetectResults.length} / ${numberAutodetectResults}` + const countText = `${goodRecords.length} / ${numberAutodetectResults}` return { circleText, percentage, countText } } - return { circleText: 0, percentage: 0, countText: `0 / ${numberAutodetectResults}` } - }, [autodetectResults, name.value, name.valueFriendly, numberAutodetectResults]) + return { circleText: `0%`, percentage: 0, countText: `0 / ${numberAutodetectResults}` } + }, [records, name.value, name.valueFriendly, numberAutodetectResults]) return ( <> diff --git a/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx b/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx index 1b7330254..4b29d75ac 100644 --- a/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx +++ b/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx @@ -1,13 +1,12 @@ -import { isNil, sortBy } from 'lodash' +import { get, isNil, sortBy } from 'lodash' import React, { HTMLProps, useCallback, useMemo } from 'react' import { useRecoilValue } from 'recoil' import styled from 'styled-components' import { motion } from 'framer-motion' import type { Dataset } from 'src/types' import { areDatasetsEqual } from 'src/types' -import { autodetectResultsAtom } from 'src/state/autodetect.state' -// import { datasetsAtom } from 'src/state/dataset.state' -// import { search } from 'src/helpers/search' +import { autodetectResultsAtom, filterGoodRecords, groupByDatasets } from 'src/state/autodetect.state' +import { search } from 'src/helpers/search' import { DatasetInfo } from 'src/components/Main/DatasetInfo' export const DatasetSelectorUl = styled.ul` @@ -66,7 +65,7 @@ export interface DatasetSelectorListProps extends HTMLProps { export function DatasetSelectorList({ datasets, - // searchTerm, + searchTerm, datasetHighlighted, onDatasetHighlighted, }: DatasetSelectorListProps) { @@ -74,35 +73,41 @@ export function DatasetSelectorList({ const autodetectResults = useRecoilValue(autodetectResultsAtom) - const { itemsStartWith, itemsInclude, itemsNotInclude } = useMemo(() => { + const autodetectResult = useMemo(() => { if (isNil(autodetectResults) || autodetectResults.length === 0) { return { itemsStartWith: [], itemsInclude: datasets, itemsNotInclude: [] } } + const goodRecordsByDataset = groupByDatasets(filterGoodRecords(autodetectResults)) + let itemsInclude = datasets.filter((candidate) => - autodetectResults.some((result) => result.result.dataset === candidate.path), - ) - itemsInclude = sortBy( - itemsInclude, - (dataset) => -autodetectResults.filter((result) => result.result.dataset === dataset.path).length, + Object.entries(goodRecordsByDataset).some(([dataset, _]) => dataset === candidate.path), ) + itemsInclude = sortBy(itemsInclude, (dataset) => -get(goodRecordsByDataset, dataset.path, []).length) + const itemsNotInclude = datasets.filter((candidate) => !itemsInclude.map((it) => it.path).includes(candidate.path)) return { itemsStartWith: [], itemsInclude, itemsNotInclude } }, [autodetectResults, datasets]) - // const { itemsStartWith, itemsInclude, itemsNotInclude } = useMemo(() => { - // if (searchTerm.trim().length === 0) { - // return { itemsStartWith: datasets, itemsInclude: [], itemsNotInclude: [] } - // } - // - // return search(datasets, searchTerm, (dataset) => [ - // dataset.attributes.name.value, - // dataset.attributes.name.valueFriendly ?? '', - // dataset.attributes.reference.value, - // ]) - // }, [datasets, searchTerm]) + const searchResult = useMemo(() => { + if (searchTerm.trim().length === 0) { + return autodetectResult + } + + return search( + [...autodetectResult.itemsStartWith, ...autodetectResult.itemsInclude, ...autodetectResult.itemsNotInclude], + searchTerm, + (dataset) => [ + dataset.attributes.name.value, + dataset.attributes.name.valueFriendly ?? '', + dataset.attributes.reference.value, + ], + ) + }, [autodetectResult, searchTerm]) + + const { itemsStartWith, itemsInclude, itemsNotInclude } = searchResult return ( diff --git a/packages_rs/nextclade-web/src/state/autodetect.state.ts b/packages_rs/nextclade-web/src/state/autodetect.state.ts index 11913819c..7b921bf9c 100644 --- a/packages_rs/nextclade-web/src/state/autodetect.state.ts +++ b/packages_rs/nextclade-web/src/state/autodetect.state.ts @@ -1,6 +1,9 @@ +/* eslint-disable no-loops/no-loops */ +import copy from 'fast-copy' +import unique from 'fork-ts-checker-webpack-plugin/lib/utils/array/unique' import { isNil } from 'lodash' import { atom, atomFamily, DefaultValue, selector, selectorFamily } from 'recoil' -import type { MinimizerIndexJson, MinimizerSearchRecord } from 'src/types' +import type { MinimizerIndexJson, MinimizerSearchRecord, MinimizerSearchResult } from 'src/types' import { isDefaultValue } from 'src/state/utils/isDefaultValue' export const minimizerIndexAtom = atom({ @@ -48,6 +51,37 @@ export const autodetectResultByIndexAtom = selectorFamily score >= 0.3 && nHits >= 10) +} + +export function filterGoodRecords(records: MinimizerSearchRecord[]) { + return records + .map((record) => { + const recordCopy = copy(record) + recordCopy.result.datasets = filterGoodDatasets(record.result) + return recordCopy + }) + .filter((record) => record.result.datasets.length > 0) +} + +export function filterBadRecords(records: MinimizerSearchRecord[]) { + const goodRecords = filterGoodRecords(records) + return records.filter( + (record) => !goodRecords.every((goodRecord) => goodRecord.fastaRecord.index === record.fastaRecord.index), + ) +} + +export function groupByDatasets(records: MinimizerSearchRecord[]) { + const names = unique(records.flatMap((record) => record.result.datasets.map((dataset) => dataset.name))) + let byDataset = {} + for (const name of names) { + const selectedRecords = records.find((record) => record.result.datasets.some((dataset) => dataset.name === name)) + byDataset = { ...byDataset, [name]: selectedRecords } + } + return byDataset +} + // Select autodetect results by dataset name export const autodetectResultsByDatasetAtom = selectorFamily({ key: 'autodetectResultByDatasetAtom', @@ -55,17 +89,18 @@ export const autodetectResultsByDatasetAtom = selectorFamily ({ get }): MinimizerSearchRecord[] | undefined => { - const results = get(autodetectResultsAtom) - if (isNil(results)) { + const records = get(autodetectResultsAtom) + if (isNil(records)) { return undefined } - return results.filter((result) => { - if (datasetId === DATASET_ID_UNDETECTED) { - return isNil(result.result.dataset) - } - return result.result.dataset === datasetId - }) + if (datasetId === DATASET_ID_UNDETECTED) { + return filterBadRecords(records) + } + + return filterGoodRecords(records).filter((record) => + record.result.datasets.some((dataset) => dataset.name === datasetId), + ) }, }) diff --git a/packages_rs/nextclade-web/src/wasm/seq_autodetect.rs b/packages_rs/nextclade-web/src/wasm/seq_autodetect.rs index 5219ba526..035f51be6 100644 --- a/packages_rs/nextclade-web/src/wasm/seq_autodetect.rs +++ b/packages_rs/nextclade-web/src/wasm/seq_autodetect.rs @@ -33,7 +33,6 @@ impl Default for NextcladeSeqAutodetectWasmParams { #[wasm_bindgen] pub struct NextcladeSeqAutodetectWasm { minimizer_index: MinimizerIndexJson, - search_params: NextcladeSeqSortParams, run_params: NextcladeSeqAutodetectWasmParams, } @@ -43,7 +42,6 @@ impl NextcladeSeqAutodetectWasm { let minimizer_index = jserr(MinimizerIndexJson::from_str(minimizer_index_json_str))?; Ok(Self { minimizer_index, - search_params: NextcladeSeqSortParams::default(), run_params: jserr(json_parse(params))?, }) } @@ -51,6 +49,8 @@ impl NextcladeSeqAutodetectWasm { pub fn autodetect(&self, qry_fasta_str: &str, callback: &js_sys::Function) -> Result<(), JsError> { let mut reader = jserr(FastaReader::from_str(&qry_fasta_str).wrap_err_with(|| "When creating fasta reader"))?; + let search_params = NextcladeSeqSortParams::default(); + let mut batch = vec![]; let mut last_flush = date_now(); @@ -62,7 +62,7 @@ impl NextcladeSeqAutodetectWasm { } let result = jserr( - run_minimizer_search(&fasta_record, &self.minimizer_index, &self.search_params).wrap_err_with(|| { + run_minimizer_search(&fasta_record, &self.minimizer_index).wrap_err_with(|| { format!( "When processing sequence #{} '{}'", fasta_record.index, fasta_record.seq_name @@ -70,7 +70,9 @@ impl NextcladeSeqAutodetectWasm { }), )?; - batch.push(MinimizerSearchRecord { fasta_record, result }); + if result.max_score >= search_params.min_score && result.total_hits >= search_params.min_hits { + batch.push(MinimizerSearchRecord { fasta_record, result }); + } if (date_now() - last_flush >= Duration::milliseconds(self.run_params.batch_interval_ms)) || batch.len() >= self.run_params.max_batch_size diff --git a/packages_rs/nextclade-web/src/workers/nextcladeAutodetect.worker.ts b/packages_rs/nextclade-web/src/workers/nextcladeAutodetect.worker.ts index 3343331b7..cc042309e 100644 --- a/packages_rs/nextclade-web/src/workers/nextcladeAutodetect.worker.ts +++ b/packages_rs/nextclade-web/src/workers/nextcladeAutodetect.worker.ts @@ -10,8 +10,8 @@ import { NextcladeSeqAutodetectWasm } from 'src/gen/nextclade-wasm' const gSubject = new Subject() -function onResultParsed(resStr: MinimizerSearchRecord[]) { - gSubject.next(resStr) +function onResultParsed(res: MinimizerSearchRecord[]) { + gSubject.next(res) } /** diff --git a/packages_rs/nextclade/Cargo.toml b/packages_rs/nextclade/Cargo.toml index 3e60a046d..8fabe54c6 100644 --- a/packages_rs/nextclade/Cargo.toml +++ b/packages_rs/nextclade/Cargo.toml @@ -44,6 +44,7 @@ num = "=0.4.0" num-traits = "=0.2.15" num_cpus = "=1.16.0" optfield = "=0.3.0" +ordered-float = { version = "=3.9.1", features = ["rand", "serde", "schemars"] } owo-colors = "=3.5.0" pretty_assertions = "=1.3.0" rayon = "=1.7.0" diff --git a/packages_rs/nextclade/src/sort/minimizer_search.rs b/packages_rs/nextclade/src/sort/minimizer_search.rs index afd136d2c..d580e82ce 100644 --- a/packages_rs/nextclade/src/sort/minimizer_search.rs +++ b/packages_rs/nextclade/src/sort/minimizer_search.rs @@ -1,20 +1,27 @@ use crate::io::fasta::FastaRecord; use crate::sort::minimizer_index::{MinimizerIndexJson, MinimizerIndexParams}; -use crate::sort::params::NextcladeSeqSortParams; use eyre::Report; -use itertools::Itertools; +use itertools::{izip, Itertools}; +use ordered_float::OrderedFloat; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::str::FromStr; +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct MinimizerSearchDatasetResult { + pub name: String, + pub length: i64, + pub n_hits: u64, + pub score: f64, +} + #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct MinimizerSearchResult { - pub dataset: Option, - pub hit_counts: Vec, pub total_hits: u64, - pub normalized_hits: Vec, - pub max_normalized_hit: f64, + pub max_score: f64, + pub datasets: Vec, } #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] @@ -28,7 +35,6 @@ pub struct MinimizerSearchRecord { pub fn run_minimizer_search( fasta_record: &FastaRecord, index: &MinimizerIndexJson, - params: &NextcladeSeqSortParams, ) -> Result { let normalization = &index.normalization; let n_refs = index.references.len(); @@ -44,36 +50,31 @@ pub fn run_minimizer_search( } // we expect hits to be proportional to the length of the sequence and the number of minimizers per reference - let mut normalized_hits: Vec = vec![0.0; hit_counts.len()]; + let mut scores: Vec = vec![0.0; hit_counts.len()]; for i in 0..n_refs { - normalized_hits[i] = hit_counts[i] as f64 * normalization[i] / fasta_record.seq.len() as f64; + scores[i] = hit_counts[i] as f64 * normalization[i] / fasta_record.seq.len() as f64; } - // require at least 30% of the maximal hits and at least 10 hits - let max_normalized_hit = normalized_hits.iter().copied().fold(0.0, f64::max); + let max_score = scores.iter().copied().fold(0.0, f64::max); let total_hits: u64 = hit_counts.iter().sum(); - if max_normalized_hit < params.min_normalized_hit || total_hits < params.min_total_hits { - Ok(MinimizerSearchResult { - dataset: None, - hit_counts, - total_hits, - normalized_hits, - max_normalized_hit, - }) - } else { - let i_ref = normalized_hits - .iter() - .position_max_by(|x, y| x.total_cmp(y)) - .expect("The `normalized_hits` cannot be empty."); - let reference = &index.references[i_ref]; - Ok(MinimizerSearchResult { - dataset: Some(reference.name.clone()), - hit_counts, - total_hits, - normalized_hits, - max_normalized_hit, + + let datasets = izip!(&index.references, hit_counts, scores) + .filter_map(|(ref_info, n_hits, score)| { + (n_hits > 0 && score >= 0.01).then_some(MinimizerSearchDatasetResult { + name: ref_info.name.clone(), + length: ref_info.length, + n_hits, + score, + }) }) - } + .sorted_by_key(|result| -OrderedFloat(result.score)) + .collect_vec(); + + Ok(MinimizerSearchResult { + total_hits, + max_score, + datasets, + }) } const fn invertible_hash(x: u64) -> u64 { diff --git a/packages_rs/nextclade/src/sort/params.rs b/packages_rs/nextclade/src/sort/params.rs index 999f9c428..a3a93f264 100644 --- a/packages_rs/nextclade/src/sort/params.rs +++ b/packages_rs/nextclade/src/sort/params.rs @@ -6,23 +6,23 @@ use serde::{Deserialize, Serialize}; #[derive(Parser, Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct NextcladeSeqSortParams { - /// Minimum value of the normalized index hit being considered for assignment + /// Minimum value of the score being considered for a detection #[clap(long)] - #[clap(default_value_t = NextcladeSeqSortParams::default().min_normalized_hit)] - pub min_normalized_hit: f64, + #[clap(default_value_t = NextcladeSeqSortParams::default().min_score)] + pub min_score: f64, - /// Minimum number of the index hits required for assignment + /// Minimum number of the index hits required for a detection #[clap(long)] - #[clap(default_value_t = NextcladeSeqSortParams::default().min_total_hits)] - pub min_total_hits: u64, + #[clap(default_value_t = NextcladeSeqSortParams::default().min_hits)] + pub min_hits: u64, } #[allow(clippy::derivable_impls)] impl Default for NextcladeSeqSortParams { fn default() -> Self { Self { - min_normalized_hit: 0.3, - min_total_hits: 10, + min_score: 0.3, + min_hits: 10, } } } From ac91ecc8631410d58d070f973654e95f12c96b03 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Fri, 8 Sep 2023 07:13:34 +0200 Subject: [PATCH 25/51] refactor: extract function --- packages_rs/nextclade-web/src/wasm/jserr.rs | 7 ++++++- .../nextclade-web/src/wasm/seq_autodetect.rs | 19 +++++++++---------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/packages_rs/nextclade-web/src/wasm/jserr.rs b/packages_rs/nextclade-web/src/wasm/jserr.rs index a581af4aa..0a7b13b8d 100644 --- a/packages_rs/nextclade-web/src/wasm/jserr.rs +++ b/packages_rs/nextclade-web/src/wasm/jserr.rs @@ -1,8 +1,13 @@ use eyre::Report; use nextclade::utils::error::report_to_string; -use wasm_bindgen::JsError; +use wasm_bindgen::{JsError, JsValue}; /// Converts Result's Err variant from eyre::Report to wasm_bindgen::JsError pub fn jserr(result: Result) -> Result { result.map_err(|report| JsError::new(&report_to_string(&report))) } + +/// Converts Result's Err variant from eyre::Report to wasm_bindgen::JsError +pub fn jserr2(result: Result) -> Result { + result.map_err(|err_val| JsError::new(&format!("{err_val:#?}"))) +} diff --git a/packages_rs/nextclade-web/src/wasm/seq_autodetect.rs b/packages_rs/nextclade-web/src/wasm/seq_autodetect.rs index 035f51be6..b3ba64483 100644 --- a/packages_rs/nextclade-web/src/wasm/seq_autodetect.rs +++ b/packages_rs/nextclade-web/src/wasm/seq_autodetect.rs @@ -1,4 +1,4 @@ -use crate::wasm::jserr::jserr; +use crate::wasm::jserr::{jserr, jserr2}; use chrono::Duration; use eyre::WrapErr; use nextclade::io::fasta::{FastaReader, FastaRecord}; @@ -77,21 +77,20 @@ impl NextcladeSeqAutodetectWasm { if (date_now() - last_flush >= Duration::milliseconds(self.run_params.batch_interval_ms)) || batch.len() >= self.run_params.max_batch_size { - let result_js = serde_wasm_bindgen::to_value(&batch)?; - callback - .call1(&JsValue::null(), &result_js) - .map_err(|err_val| JsError::new(&format!("{err_val:#?}")))?; + self.flush_batch(callback, &mut batch)?; last_flush = date_now(); - batch.clear(); } } + self.flush_batch(callback, &mut batch)?; + + Ok(()) + } + + fn flush_batch(&self, callback: &js_sys::Function, batch: &mut Vec) -> Result<(), JsError> { let result_js = serde_wasm_bindgen::to_value(&batch)?; - callback - .call1(&JsValue::null(), &result_js) - .map_err(|err_val| JsError::new(&format!("{err_val:#?}")))?; + jserr2(callback.call1(&JsValue::null(), &result_js))?; batch.clear(); - Ok(()) } } From af4841e8b34ffa7a12e8ff5d413e5e4feb6abf17 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Fri, 8 Sep 2023 07:31:06 +0200 Subject: [PATCH 26/51] refactor: lint --- packages_rs/nextclade-web/src/helpers/string.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages_rs/nextclade-web/src/helpers/string.ts b/packages_rs/nextclade-web/src/helpers/string.ts index 531c4fdfb..05647c8f8 100644 --- a/packages_rs/nextclade-web/src/helpers/string.ts +++ b/packages_rs/nextclade-web/src/helpers/string.ts @@ -43,5 +43,5 @@ export function findSimilarStrings(haystack: string[], needle: string): string[] } export function firstLetter(s: string): string | undefined { - return s.split('').find((c) => c.match(/[A-z]/)) + return s.split('').find((c) => c.toLowerCase().match(/[a-z]/)) } From d136b8487e20ae1db696978dcff40c9230239dc5 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Fri, 8 Sep 2023 07:44:21 +0200 Subject: [PATCH 27/51] refactor(cli): simplify check for completions cli args --- packages_rs/nextclade-cli/src/cli/nextclade_cli.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs b/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs index d82056a69..b51e8615b 100644 --- a/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs +++ b/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs @@ -29,13 +29,6 @@ lazy_static! { pub static ref SHELLS: Vec<&'static str> = ["bash", "elvish", "fish", "fig", "powershell", "zsh"].to_vec(); } -pub fn check_shells(value: &str) -> Result { - SHELLS - .contains(&value) - .then_some(value.to_owned()) - .ok_or_else(|| eyre!("Unknown shell: '{value}'. Possible values: {}", SHELLS.join(", "))) -} - #[derive(Parser, Debug)] #[clap(name = "nextclade")] #[clap(author, version)] @@ -71,7 +64,7 @@ pub enum NextcladeCommands { /// Completions { /// Name of the shell to generate appropriate completions - #[clap(value_name = "SHELL", default_value_t = String::from("bash"), value_parser = check_shells)] + #[clap(value_name = "SHELL", default_value_t = String::from("bash"), value_parser = SHELLS.clone())] shell: String, }, From e33c04f14cea40125140cb47d228205b5d157e78 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Fri, 8 Sep 2023 07:45:57 +0200 Subject: [PATCH 28/51] feat(cli): rename 'seq sort' subcommand to 'sort' --- .../nextclade-cli/src/cli/nextclade_cli.rs | 31 +++++-------------- .../src/cli/nextclade_seq_sort.rs | 14 ++++----- 2 files changed, 14 insertions(+), 31 deletions(-) diff --git a/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs b/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs index b51e8615b..49f53f5bb 100644 --- a/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs +++ b/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs @@ -68,20 +68,20 @@ pub enum NextcladeCommands { shell: String, }, - /// Run alignment, mutation calling, clade assignment, quality checks and phylogenetic placement + /// Run sequence analysis: alignment, mutation calling, clade assignment, quality checks and phylogenetic placement /// /// For short help type: `nextclade -h`, for extended help type: `nextclade --help`. Each subcommand has its own help, for example: `nextclade run --help`. Run(Box), - /// List and download available Nextclade datasets + /// List and download available Nextclade datasets (pathogens) /// /// For short help type: `nextclade -h`, for extended help type: `nextclade --help`. Each subcommand has its own help, for example: `nextclade dataset --help`. Dataset(Box), - /// Perform operations on sequences + /// Sort sequences according to the inferred Nextclade dataset (pathogen) /// - /// For short help type: `nextclade -h`, for extended help type: `nextclade --help`. Each subcommand has its own help, for example: `nextclade seq --help`. - Seq(Box), + /// For short help type: `nextclade -h`, for extended help type: `nextclade --help`. Each subcommand has its own help, for example: `nextclade sort --help`. + Sort(Box), } #[derive(Parser, Debug)] @@ -618,25 +618,10 @@ pub struct NextcladeRunArgs { pub other_params: NextcladeRunOtherParams, } -#[derive(Parser, Debug)] -pub struct NextcladeSeqArgs { - #[clap(subcommand)] - pub command: NextcladeSeqCommands, -} - -#[derive(Subcommand, Debug)] -#[clap(verbatim_doc_comment)] -pub enum NextcladeSeqCommands { - /// Group (sort) input sequences according to the inferred dataset (pathogen) - /// - /// For short help type: `nextclade -h`, for extended help type: `nextclade --help`. Each subcommand has its own help, for example: `nextclade seq sort --help`. - Sort(NextcladeSeqSortArgs), -} - #[allow(clippy::struct_excessive_bools)] #[derive(Parser, Debug)] #[clap(verbatim_doc_comment)] -pub struct NextcladeSeqSortArgs { +pub struct NextcladeSortArgs { /// Path to one or multiple FASTA files with input sequences /// /// Supports the following compression formats: "gz", "bz2", "xz", "zst". If no files provided, the plain fasta input is read from standard input (stdin). @@ -993,8 +978,6 @@ pub fn nextclade_parse_cli_args() -> Result<(), Report> { NextcladeDatasetCommands::List(dataset_list_args) => nextclade_dataset_list(dataset_list_args), NextcladeDatasetCommands::Get(dataset_get_args) => nextclade_dataset_get(&dataset_get_args), }, - NextcladeCommands::Seq(seq_command) => match seq_command.command { - NextcladeSeqCommands::Sort(seq_sort_args) => nextclade_seq_sort(&seq_sort_args), - }, + NextcladeCommands::Sort(seq_sort_args) => nextclade_seq_sort(&seq_sort_args), } } diff --git a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs index 8e071f018..8b5c99aef 100644 --- a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs +++ b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs @@ -1,4 +1,4 @@ -use crate::cli::nextclade_cli::{NextcladeRunOtherParams, NextcladeSeqSortArgs}; +use crate::cli::nextclade_cli::{NextcladeRunOtherParams, NextcladeSortArgs}; use crate::dataset::dataset_download::download_datasets_index_json; use crate::io::http_client::HttpClient; use eyre::{Report, WrapErr}; @@ -15,10 +15,10 @@ use std::path::PathBuf; use std::str::FromStr; use tinytemplate::TinyTemplate; -pub fn nextclade_seq_sort(args: &NextcladeSeqSortArgs) -> Result<(), Report> { +pub fn nextclade_seq_sort(args: &NextcladeSortArgs) -> Result<(), Report> { check_args(args)?; - let NextcladeSeqSortArgs { + let NextcladeSortArgs { server, proxy_config, input_minimizer_index_json, @@ -62,8 +62,8 @@ pub fn nextclade_seq_sort(args: &NextcladeSeqSortArgs) -> Result<(), Report> { run(args, &minimizer_index) } -pub fn run(args: &NextcladeSeqSortArgs, minimizer_index: &MinimizerIndexJson) -> Result<(), Report> { - let NextcladeSeqSortArgs { +pub fn run(args: &NextcladeSortArgs, minimizer_index: &MinimizerIndexJson) -> Result<(), Report> { + let NextcladeSortArgs { input_fastas, output_dir, output, @@ -200,8 +200,8 @@ struct OutputTemplateContext<'a> { name: &'a str, } -fn check_args(args: &NextcladeSeqSortArgs) -> Result<(), Report> { - let NextcladeSeqSortArgs { output_dir, output, .. } = args; +fn check_args(args: &NextcladeSortArgs) -> Result<(), Report> { + let NextcladeSortArgs { output_dir, output, .. } = args; if output.is_some() && output_dir.is_some() { return make_error!( From bfa0e9bed917dbc0766439c76b7bdd344d670199 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Fri, 8 Sep 2023 08:38:33 +0200 Subject: [PATCH 29/51] refactor: remove unused import --- packages_rs/nextclade/src/gene/gene_map_display.rs | 1 - packages_rs/nextclade/src/io/nextclade_csv.rs | 1 - packages_rs/nextclade/src/tree/split_muts.rs | 1 - packages_rs/nextclade/src/tree/split_muts2.rs | 1 - packages_rs/nextclade/src/tree/tree_builder.rs | 1 - 5 files changed, 5 deletions(-) diff --git a/packages_rs/nextclade/src/gene/gene_map_display.rs b/packages_rs/nextclade/src/gene/gene_map_display.rs index 2158fe64d..10079b49b 100644 --- a/packages_rs/nextclade/src/gene/gene_map_display.rs +++ b/packages_rs/nextclade/src/gene/gene_map_display.rs @@ -10,7 +10,6 @@ use eyre::Report; use itertools::{max as iter_max, Itertools}; use num_traits::clamp; use owo_colors::OwoColorize; -use regex::internal::Input; use std::cmp::{max, min}; use std::io::Write; diff --git a/packages_rs/nextclade/src/io/nextclade_csv.rs b/packages_rs/nextclade/src/io/nextclade_csv.rs index 676b736a7..b01cba894 100644 --- a/packages_rs/nextclade/src/io/nextclade_csv.rs +++ b/packages_rs/nextclade/src/io/nextclade_csv.rs @@ -24,7 +24,6 @@ use eyre::Report; use indexmap::{indexmap, IndexMap}; use itertools::{chain, Either, Itertools}; use lazy_static::lazy_static; -use regex::internal::Input; use serde::{Deserialize, Serialize}; use std::borrow::Cow; use std::fmt::Display; diff --git a/packages_rs/nextclade/src/tree/split_muts.rs b/packages_rs/nextclade/src/tree/split_muts.rs index 50493824e..9bcf3a5e9 100644 --- a/packages_rs/nextclade/src/tree/split_muts.rs +++ b/packages_rs/nextclade/src/tree/split_muts.rs @@ -6,7 +6,6 @@ use crate::coord::position::PositionLike; use crate::make_internal_error; use eyre::{Report, WrapErr}; use itertools::{chain, Itertools}; -use regex::internal::Input; use std::collections::BTreeMap; use std::fmt::Display; use std::hash::Hash; diff --git a/packages_rs/nextclade/src/tree/split_muts2.rs b/packages_rs/nextclade/src/tree/split_muts2.rs index d1dd46fab..5fe4ab0f0 100644 --- a/packages_rs/nextclade/src/tree/split_muts2.rs +++ b/packages_rs/nextclade/src/tree/split_muts2.rs @@ -4,7 +4,6 @@ use crate::analyze::nuc_del::NucDel; use crate::analyze::nuc_sub::NucSub; use crate::tree::split_muts::SplitMutsResult; use itertools::Itertools; -use regex::internal::Input; use std::collections::{BTreeMap, HashSet}; /// Split mutations into 3 groups: diff --git a/packages_rs/nextclade/src/tree/tree_builder.rs b/packages_rs/nextclade/src/tree/tree_builder.rs index 6880458a6..e60b7b39d 100644 --- a/packages_rs/nextclade/src/tree/tree_builder.rs +++ b/packages_rs/nextclade/src/tree/tree_builder.rs @@ -15,7 +15,6 @@ use crate::types::outputs::NextcladeOutputs; use crate::utils::collections::concat_to_vec; use eyre::{Report, WrapErr}; use itertools::Itertools; -use regex::internal::Input; use std::collections::BTreeMap; pub fn graph_attach_new_nodes_in_place( From e30adc09688ee784efb462fcbda1420b94afc2b3 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Fri, 8 Sep 2023 11:15:09 +0200 Subject: [PATCH 30/51] feat(cli): write sorted sequences to all common path prefixes --- .../src/cli/nextclade_seq_sort.rs | 153 ++++++++++-------- .../nextclade/benches/bench_create_stripes.rs | 7 +- packages_rs/nextclade/src/io/fs.rs | 8 + 3 files changed, 100 insertions(+), 68 deletions(-) diff --git a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs index 8b5c99aef..32648c976 100644 --- a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs +++ b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs @@ -3,15 +3,18 @@ use crate::dataset::dataset_download::download_datasets_index_json; use crate::io::http_client::HttpClient; use eyre::{Report, WrapErr}; use itertools::Itertools; -use log::{info, trace, LevelFilter}; +use log::{info, LevelFilter}; use nextclade::io::fasta::{FastaReader, FastaRecord, FastaWriter}; +use nextclade::io::fs::path_to_string; use nextclade::make_error; use nextclade::sort::minimizer_index::{MinimizerIndexJson, MINIMIZER_INDEX_ALGO_VERSION}; use nextclade::sort::minimizer_search::{run_minimizer_search, MinimizerSearchRecord}; +use nextclade::utils::option::OptionMapRefFallible; use nextclade::utils::string::truncate; use serde::Serialize; +use std::collections::btree_map::Entry::{Occupied, Vacant}; use std::collections::BTreeMap; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::str::FromStr; use tinytemplate::TinyTemplate; @@ -112,9 +115,7 @@ pub fn run(args: &NextcladeSortArgs, minimizer_index: &MinimizerIndexJson) -> Re }) .unwrap(); - if result.max_score >= search_params.min_score - && result.total_hits >= search_params.min_hits - { + if result.max_score >= search_params.min_score && result.total_hits >= search_params.min_hits { result_sender .send(MinimizerSearchRecord { fasta_record, result }) .wrap_err("When sending minimizer record into the channel") @@ -129,72 +130,98 @@ pub fn run(args: &NextcladeSortArgs, minimizer_index: &MinimizerIndexJson) -> Re let writer = s.spawn(move || { let output_dir = &output_dir; let output = &output; + writer_thread(output, output_dir, result_receiver).unwrap(); + }); + }); - let tt = output.as_ref().map(move |output| { - let mut tt = TinyTemplate::new(); - tt.add_template("output", output) - .wrap_err_with(|| format!("When parsing template: {output}")) - .unwrap(); - tt - }); - - println!( - "{:40} | {:40} | {:10} | {:10}", - "Seq. name", "dataset", "total hits", "max hit" - ); - - let mut writers = BTreeMap::new(); - - for record in result_receiver { - let dataset = record.result.datasets.first(); - - if let Some(dataset) = dataset { - let name = &dataset.name; - - let filepath = match (&tt, output_dir) { - (Some(tt), None) => { - let filepath_str = tt - .render("output", &OutputTemplateContext { name }) - .wrap_err("When rendering output path template") - .unwrap(); + Ok(()) +} - Some( - PathBuf::from_str(&filepath_str) - .wrap_err_with(|| format!("Invalid output translations path: '{filepath_str}'")) - .unwrap(), - ) - } - (None, Some(output_dir)) => Some(output_dir.join(name).join("sequences.fasta")), - _ => None, - }; - - if let Some(filepath) = filepath { - let writer = writers.entry(filepath.clone()).or_insert_with(|| { - trace!("Creating fasta writer to file {filepath:#?}"); - FastaWriter::from_path(filepath).unwrap() - }); - - writer - .write(&record.fasta_record.seq_name, &record.fasta_record.seq, false) - .unwrap(); - } +fn writer_thread( + output: &Option, + output_dir: &Option, + result_receiver: crossbeam_channel::Receiver, +) -> Result<(), Report> { + let template = output.map_ref_fallible(move |output| -> Result { + let mut template = TinyTemplate::new(); + template + .add_template("output", output) + .wrap_err_with(|| format!("When parsing template: {output}"))?; + Ok(template) + })?; + + println!( + "{:40} | {:40} | {:10} | {:10}", + "Seq. name", "dataset", "total hits", "max hit" + ); + + let mut writers = BTreeMap::new(); + + for record in result_receiver { + for dataset in &record.result.datasets { + let name = &dataset.name; + + let names = name + .split('/') + .scan(PathBuf::new(), |name, component| { + *name = name.join(component); + Some(name.clone()) + }) + .map(path_to_string) + .collect::, Report>>()?; + + for name in names { + let filepath = get_filepath(&name, &template, output_dir)?; + + if let Some(filepath) = filepath { + let writer = get_or_insert_writer(&mut writers, filepath)?; + writer.write(&record.fasta_record.seq_name, &record.fasta_record.seq, false)?; } - - let name_or_empty = dataset.as_ref().map(|dataset| dataset.name.clone()).unwrap_or_default(); - println!( - "{:40} | {:40} | {:>10} | {:>.3}", - &truncate(record.fasta_record.seq_name, 40), - &truncate(name_or_empty, 40), - &record.result.total_hits, - &record.result.max_score - ); } - }); - }); + } + + let dataset = record.result.datasets.first(); + let name_or_empty = dataset.as_ref().map(|dataset| dataset.name.clone()).unwrap_or_default(); + println!( + "{:40} | {:40} | {:>10} | {:>.3}", + &truncate(record.fasta_record.seq_name, 40), + &truncate(name_or_empty, 40), + &record.result.total_hits, + &record.result.max_score + ); + } Ok(()) } +fn get_or_insert_writer( + writers: &mut BTreeMap, + filepath: impl AsRef, +) -> Result<&mut FastaWriter, Report> { + Ok(match writers.entry(filepath.as_ref().to_owned()) { + Occupied(e) => e.into_mut(), + Vacant(e) => e.insert(FastaWriter::from_path(filepath)?), + }) +} + +fn get_filepath( + name: &str, + tt: &Option, + output_dir: &Option, +) -> Result, Report> { + Ok(match (&tt, output_dir) { + (Some(tt), None) => { + let filepath_str = tt + .render("output", &OutputTemplateContext { name }) + .wrap_err("When rendering output path template")?; + + Some(PathBuf::from_str(&filepath_str).wrap_err_with(|| format!("Invalid output path: '{filepath_str}'"))?) + } + (None, Some(output_dir)) => Some(output_dir.join(name).join("sequences.fasta")), + _ => None, + }) +} + #[derive(Serialize)] struct OutputTemplateContext<'a> { name: &'a str, diff --git a/packages_rs/nextclade/benches/bench_create_stripes.rs b/packages_rs/nextclade/benches/bench_create_stripes.rs index e10348d55..29fae0e7f 100644 --- a/packages_rs/nextclade/benches/bench_create_stripes.rs +++ b/packages_rs/nextclade/benches/bench_create_stripes.rs @@ -22,9 +22,7 @@ pub fn bench_create_stripes(c: &mut Criterion) { let excess_bandwidth = black_box(2); let qry_len = black_box(30); let ref_len = black_box(40); - let max_indel = black_box(400); - let allowed_mismatches = black_box(2); - let max_band_area = black_box(500_000_000); + let minimal_bandwidth = black_box(1); let mut group = c.benchmark_group("create_stripes"); group.throughput(Throughput::Bytes(qry_len as u64)); @@ -36,8 +34,7 @@ pub fn bench_create_stripes(c: &mut Criterion) { ref_len, terminal_bandwidth, excess_bandwidth, - allowed_mismatches, - max_band_area, + minimal_bandwidth, ) }); }); diff --git a/packages_rs/nextclade/src/io/fs.rs b/packages_rs/nextclade/src/io/fs.rs index 4120049a9..6c2aaff32 100644 --- a/packages_rs/nextclade/src/io/fs.rs +++ b/packages_rs/nextclade/src/io/fs.rs @@ -89,3 +89,11 @@ pub fn read_reader_to_string(reader: impl Read) -> Result { reader.read_to_string(&mut data)?; Ok(data) } + +pub fn path_to_string(p: impl AsRef) -> Result { + p.as_ref() + .as_os_str() + .to_str() + .map(ToOwned::to_owned) + .ok_or_else(|| eyre!("Unable to convert path to string: {:#?}", p.as_ref())) +} From ac7c2d50cdec96824285cc35c141bb629b11ca85 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Fri, 8 Sep 2023 11:23:50 +0200 Subject: [PATCH 31/51] feat(cli): prettify help --- packages_rs/nextclade-cli/src/cli/nextclade_cli.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs b/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs index 40259efde..1ace36843 100644 --- a/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs +++ b/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs @@ -684,10 +684,10 @@ pub struct NextcladeSortArgs { #[clap(group = "outputs")] pub output: Option, - #[clap(flatten, next_help_heading = " Algorithm")] + #[clap(flatten, next_help_heading = "Algorithm")] pub search_params: NextcladeSeqSortParams, - #[clap(flatten, next_help_heading = " Other")] + #[clap(flatten, next_help_heading = "Other")] pub other_params: NextcladeRunOtherParams, /// Use custom dataset server. From 61eef71cdf85a6b44cdab28c79cf813c9f11acff Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Fri, 8 Sep 2023 11:47:53 +0200 Subject: [PATCH 32/51] feat(cli): print all dataset detections in terminal output --- Cargo.lock | 1 + packages_rs/nextclade-cli/Cargo.toml | 1 + .../src/cli/nextclade_seq_sort.rs | 37 +++++++++++++------ 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 991b436fd..482e7bd77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1787,6 +1787,7 @@ dependencies = [ "log", "nextclade", "num_cpus", + "ordered-float", "owo-colors", "pretty_assertions", "rayon", diff --git a/packages_rs/nextclade-cli/Cargo.toml b/packages_rs/nextclade-cli/Cargo.toml index c87a9b83f..b82b3e730 100644 --- a/packages_rs/nextclade-cli/Cargo.toml +++ b/packages_rs/nextclade-cli/Cargo.toml @@ -28,6 +28,7 @@ lazy_static = "=1.4.0" log = "=0.4.19" nextclade = { path = "../nextclade" } num_cpus = "=1.16.0" +ordered-float = { version = "=3.9.1", features = ["rand", "serde", "schemars"] } owo-colors = "=3.5.0" pretty_assertions = "=1.3.0" rayon = "=1.7.0" diff --git a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs index 32648c976..383eac35a 100644 --- a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs +++ b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs @@ -11,6 +11,7 @@ use nextclade::sort::minimizer_index::{MinimizerIndexJson, MINIMIZER_INDEX_ALGO_ use nextclade::sort::minimizer_search::{run_minimizer_search, MinimizerSearchRecord}; use nextclade::utils::option::OptionMapRefFallible; use nextclade::utils::string::truncate; +use ordered_float::OrderedFloat; use serde::Serialize; use std::collections::btree_map::Entry::{Occupied, Vacant}; use std::collections::BTreeMap; @@ -150,15 +151,25 @@ fn writer_thread( Ok(template) })?; + println!("{}┐", "─".repeat(110)); + println!( - "{:40} | {:40} | {:10} | {:10}", - "Seq. name", "dataset", "total hits", "max hit" + "{:^40} │ {:^40} │ {:^10} │ {:^10} │", + "Sequence name", "Dataset", "Score", "Num. hits" ); + println!("{}┤", "─".repeat(110)); + let mut writers = BTreeMap::new(); for record in result_receiver { - for dataset in &record.result.datasets { + let datasets = record + .result + .datasets + .iter() + .sorted_by_key(|dataset| -OrderedFloat(dataset.score)); + + for (i, dataset) in datasets.enumerate() { let name = &dataset.name; let names = name @@ -167,6 +178,7 @@ fn writer_thread( *name = name.join(component); Some(name.clone()) }) + .unique() .map(path_to_string) .collect::, Report>>()?; @@ -178,17 +190,18 @@ fn writer_thread( writer.write(&record.fasta_record.seq_name, &record.fasta_record.seq, false)?; } } + + let name = if i == 0 { &record.fasta_record.seq_name } else { "" }; + println!( + "{:40} │ {:40} │ {:>10.3} │ {:>10} │", + &truncate(name, 40), + &truncate(&dataset.name, 40), + &dataset.score, + &dataset.n_hits, + ); } - let dataset = record.result.datasets.first(); - let name_or_empty = dataset.as_ref().map(|dataset| dataset.name.clone()).unwrap_or_default(); - println!( - "{:40} | {:40} | {:>10} | {:>.3}", - &truncate(record.fasta_record.seq_name, 40), - &truncate(name_or_empty, 40), - &record.result.total_hits, - &record.result.max_score - ); + println!("{}┤", "─".repeat(110)); } Ok(()) From eb15094ad3948b1905acd629bb25d4b51c2142da Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Tue, 12 Sep 2023 08:21:49 +0200 Subject: [PATCH 33/51] feat: filter datasets in wasm to speedup web ui --- .../src/cli/nextclade_seq_sort.rs | 31 +++++++++++------- .../src/components/Main/DatasetInfo.tsx | 9 ++---- .../components/Main/DatasetSelectorList.tsx | 8 ++--- .../src/state/autodetect.state.ts | 32 +++---------------- .../nextclade-web/src/wasm/seq_autodetect.rs | 11 ++++--- .../nextclade/src/sort/minimizer_search.rs | 4 ++- 6 files changed, 39 insertions(+), 56 deletions(-) diff --git a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs index 383eac35a..5d030137b 100644 --- a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs +++ b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs @@ -107,7 +107,7 @@ pub fn run(args: &NextcladeSortArgs, minimizer_index: &MinimizerIndexJson) -> Re for fasta_record in &fasta_receiver { info!("Processing sequence '{}'", fasta_record.seq_name); - let result = run_minimizer_search(&fasta_record, minimizer_index) + let result = run_minimizer_search(&fasta_record, minimizer_index, search_params) .wrap_err_with(|| { format!( "When processing sequence #{} '{}'", @@ -116,12 +116,10 @@ pub fn run(args: &NextcladeSortArgs, minimizer_index: &MinimizerIndexJson) -> Re }) .unwrap(); - if result.max_score >= search_params.min_score && result.total_hits >= search_params.min_hits { - result_sender - .send(MinimizerSearchRecord { fasta_record, result }) - .wrap_err("When sending minimizer record into the channel") - .unwrap(); - } + result_sender + .send(MinimizerSearchRecord { fasta_record, result }) + .wrap_err("When sending minimizer record into the channel") + .unwrap(); } drop(result_sender); @@ -167,9 +165,16 @@ fn writer_thread( .result .datasets .iter() - .sorted_by_key(|dataset| -OrderedFloat(dataset.score)); + .sorted_by_key(|dataset| -OrderedFloat(dataset.score)) + .collect_vec(); + + print!("{:40}", truncate(&record.fasta_record.seq_name, 40)); + + if datasets.is_empty() { + println!(" │ {:40} │ {:>10.3} │ {:>10} │", "", "", "",); + } - for (i, dataset) in datasets.enumerate() { + for (i, dataset) in datasets.into_iter().enumerate() { let name = &dataset.name; let names = name @@ -191,10 +196,12 @@ fn writer_thread( } } - let name = if i == 0 { &record.fasta_record.seq_name } else { "" }; + if i != 0 { + print!("{:40}", ""); + } + println!( - "{:40} │ {:40} │ {:>10.3} │ {:>10} │", - &truncate(name, 40), + " │ {:40} │ {:>10.3} │ {:>10} │", &truncate(&dataset.name, 40), &dataset.score, &dataset.n_hits, diff --git a/packages_rs/nextclade-web/src/components/Main/DatasetInfo.tsx b/packages_rs/nextclade-web/src/components/Main/DatasetInfo.tsx index 1d8b97871..eff4667f9 100644 --- a/packages_rs/nextclade-web/src/components/Main/DatasetInfo.tsx +++ b/packages_rs/nextclade-web/src/components/Main/DatasetInfo.tsx @@ -10,7 +10,6 @@ import { useTranslationSafe } from 'src/helpers/useTranslationSafe' import { autodetectResultsByDatasetAtom, DATASET_ID_UNDETECTED, - filterGoodRecords, numberAutodetectResultsAtom, } from 'src/state/autodetect.state' import type { Dataset } from 'src/types' @@ -185,12 +184,10 @@ function DatasetInfoAutodetectProgressCircle({ dataset }: DatasetInfoCircleProps } } - const goodRecords = filterGoodRecords(records) - - if (goodRecords.length > 0) { - const percentage = goodRecords.length / numberAutodetectResults + if (records.length > 0) { + const percentage = records.length / numberAutodetectResults const circleText = `${(100 * percentage).toFixed(0)}%` - const countText = `${goodRecords.length} / ${numberAutodetectResults}` + const countText = `${records.length} / ${numberAutodetectResults}` return { circleText, percentage, countText } } return { circleText: `0%`, percentage: 0, countText: `0 / ${numberAutodetectResults}` } diff --git a/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx b/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx index 4b29d75ac..d860b1cff 100644 --- a/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx +++ b/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx @@ -5,7 +5,7 @@ import styled from 'styled-components' import { motion } from 'framer-motion' import type { Dataset } from 'src/types' import { areDatasetsEqual } from 'src/types' -import { autodetectResultsAtom, filterGoodRecords, groupByDatasets } from 'src/state/autodetect.state' +import { autodetectResultsAtom, groupByDatasets } from 'src/state/autodetect.state' import { search } from 'src/helpers/search' import { DatasetInfo } from 'src/components/Main/DatasetInfo' @@ -78,13 +78,13 @@ export function DatasetSelectorList({ return { itemsStartWith: [], itemsInclude: datasets, itemsNotInclude: [] } } - const goodRecordsByDataset = groupByDatasets(filterGoodRecords(autodetectResults)) + const recordsByDataset = groupByDatasets(autodetectResults) let itemsInclude = datasets.filter((candidate) => - Object.entries(goodRecordsByDataset).some(([dataset, _]) => dataset === candidate.path), + Object.entries(recordsByDataset).some(([dataset, _]) => dataset === candidate.path), ) - itemsInclude = sortBy(itemsInclude, (dataset) => -get(goodRecordsByDataset, dataset.path, []).length) + itemsInclude = sortBy(itemsInclude, (dataset) => -get(recordsByDataset, dataset.path, []).length) const itemsNotInclude = datasets.filter((candidate) => !itemsInclude.map((it) => it.path).includes(candidate.path)) diff --git a/packages_rs/nextclade-web/src/state/autodetect.state.ts b/packages_rs/nextclade-web/src/state/autodetect.state.ts index 7b921bf9c..0c3f0b091 100644 --- a/packages_rs/nextclade-web/src/state/autodetect.state.ts +++ b/packages_rs/nextclade-web/src/state/autodetect.state.ts @@ -1,9 +1,8 @@ /* eslint-disable no-loops/no-loops */ -import copy from 'fast-copy' import unique from 'fork-ts-checker-webpack-plugin/lib/utils/array/unique' -import { isNil } from 'lodash' +import { isEmpty, isNil } from 'lodash' import { atom, atomFamily, DefaultValue, selector, selectorFamily } from 'recoil' -import type { MinimizerIndexJson, MinimizerSearchRecord, MinimizerSearchResult } from 'src/types' +import type { MinimizerIndexJson, MinimizerSearchRecord } from 'src/types' import { isDefaultValue } from 'src/state/utils/isDefaultValue' export const minimizerIndexAtom = atom({ @@ -51,27 +50,6 @@ export const autodetectResultByIndexAtom = selectorFamily score >= 0.3 && nHits >= 10) -} - -export function filterGoodRecords(records: MinimizerSearchRecord[]) { - return records - .map((record) => { - const recordCopy = copy(record) - recordCopy.result.datasets = filterGoodDatasets(record.result) - return recordCopy - }) - .filter((record) => record.result.datasets.length > 0) -} - -export function filterBadRecords(records: MinimizerSearchRecord[]) { - const goodRecords = filterGoodRecords(records) - return records.filter( - (record) => !goodRecords.every((goodRecord) => goodRecord.fastaRecord.index === record.fastaRecord.index), - ) -} - export function groupByDatasets(records: MinimizerSearchRecord[]) { const names = unique(records.flatMap((record) => record.result.datasets.map((dataset) => dataset.name))) let byDataset = {} @@ -95,12 +73,10 @@ export const autodetectResultsByDatasetAtom = selectorFamily isEmpty(record.result.datasets)) } - return filterGoodRecords(records).filter((record) => - record.result.datasets.some((dataset) => dataset.name === datasetId), - ) + return records.filter((record) => record.result.datasets.some((dataset) => dataset.name === datasetId)) }, }) diff --git a/packages_rs/nextclade-web/src/wasm/seq_autodetect.rs b/packages_rs/nextclade-web/src/wasm/seq_autodetect.rs index b3ba64483..1a3fe6c3e 100644 --- a/packages_rs/nextclade-web/src/wasm/seq_autodetect.rs +++ b/packages_rs/nextclade-web/src/wasm/seq_autodetect.rs @@ -62,7 +62,7 @@ impl NextcladeSeqAutodetectWasm { } let result = jserr( - run_minimizer_search(&fasta_record, &self.minimizer_index).wrap_err_with(|| { + run_minimizer_search(&fasta_record, &self.minimizer_index, &search_params).wrap_err_with(|| { format!( "When processing sequence #{} '{}'", fasta_record.index, fasta_record.seq_name @@ -70,11 +70,9 @@ impl NextcladeSeqAutodetectWasm { }), )?; - if result.max_score >= search_params.min_score && result.total_hits >= search_params.min_hits { - batch.push(MinimizerSearchRecord { fasta_record, result }); - } + batch.push(MinimizerSearchRecord { fasta_record, result }); - if (date_now() - last_flush >= Duration::milliseconds(self.run_params.batch_interval_ms)) + if date_now() - last_flush >= Duration::milliseconds(self.run_params.batch_interval_ms) || batch.len() >= self.run_params.max_batch_size { self.flush_batch(callback, &mut batch)?; @@ -88,6 +86,9 @@ impl NextcladeSeqAutodetectWasm { } fn flush_batch(&self, callback: &js_sys::Function, batch: &mut Vec) -> Result<(), JsError> { + if batch.is_empty() { + return Ok(()); + } let result_js = serde_wasm_bindgen::to_value(&batch)?; jserr2(callback.call1(&JsValue::null(), &result_js))?; batch.clear(); diff --git a/packages_rs/nextclade/src/sort/minimizer_search.rs b/packages_rs/nextclade/src/sort/minimizer_search.rs index d580e82ce..bc01dbf1b 100644 --- a/packages_rs/nextclade/src/sort/minimizer_search.rs +++ b/packages_rs/nextclade/src/sort/minimizer_search.rs @@ -1,5 +1,6 @@ use crate::io::fasta::FastaRecord; use crate::sort::minimizer_index::{MinimizerIndexJson, MinimizerIndexParams}; +use crate::sort::params::NextcladeSeqSortParams; use eyre::Report; use itertools::{izip, Itertools}; use ordered_float::OrderedFloat; @@ -35,6 +36,7 @@ pub struct MinimizerSearchRecord { pub fn run_minimizer_search( fasta_record: &FastaRecord, index: &MinimizerIndexJson, + search_params: &NextcladeSeqSortParams, ) -> Result { let normalization = &index.normalization; let n_refs = index.references.len(); @@ -60,7 +62,7 @@ pub fn run_minimizer_search( let datasets = izip!(&index.references, hit_counts, scores) .filter_map(|(ref_info, n_hits, score)| { - (n_hits > 0 && score >= 0.01).then_some(MinimizerSearchDatasetResult { + (n_hits >= search_params.min_hits && score >= search_params.min_score).then_some(MinimizerSearchDatasetResult { name: ref_info.name.clone(), length: ref_info.length, n_hits, From c4d2763b07d034b799f0b4c3729c5ec5d34f5124 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Tue, 12 Sep 2023 22:07:07 +0200 Subject: [PATCH 34/51] feat: remove list animations to speed things up --- packages_rs/nextclade-web/package.json | 1 - .../components/Main/DatasetSelectorList.tsx | 32 +++++-------------- packages_rs/nextclade-web/yarn.lock | 26 --------------- 3 files changed, 8 insertions(+), 51 deletions(-) diff --git a/packages_rs/nextclade-web/package.json b/packages_rs/nextclade-web/package.json index bf0a10db6..0c7446ac1 100644 --- a/packages_rs/nextclade-web/package.json +++ b/packages_rs/nextclade-web/package.json @@ -105,7 +105,6 @@ "file-saver": "2.0.5", "flag-icon-css": "3.5.0", "formik": "2.2.9", - "framer-motion": "10.16.4", "history": "5.3.0", "i18next": "19.3.2", "immutable": "4.0.0", diff --git a/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx b/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx index d860b1cff..a5c69c2d9 100644 --- a/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx +++ b/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx @@ -1,38 +1,29 @@ import { get, isNil, sortBy } from 'lodash' -import React, { HTMLProps, useCallback, useMemo } from 'react' +import React, { useCallback, useMemo } from 'react' +import { ListGroup, ListGroupItem } from 'reactstrap' import { useRecoilValue } from 'recoil' import styled from 'styled-components' -import { motion } from 'framer-motion' import type { Dataset } from 'src/types' import { areDatasetsEqual } from 'src/types' import { autodetectResultsAtom, groupByDatasets } from 'src/state/autodetect.state' import { search } from 'src/helpers/search' import { DatasetInfo } from 'src/components/Main/DatasetInfo' -export const DatasetSelectorUl = styled.ul` +export const DatasetSelectorUl = styled(ListGroup)` flex: 1; overflow-y: scroll; height: 100%; - padding: 0; ` -export const DatasetSelectorLi = styled(motion.li)<{ $active?: boolean; $isDimmed?: boolean }>` +export const DatasetSelectorLi = styled(ListGroupItem)<{ $isDimmed?: boolean }>` list-style: none; margin: 0; padding: 0.5rem; cursor: pointer; - filter: ${(props) => props.$isDimmed && !props.$active && 'invert(0.1) brightness(0.9)'}; - background-color: ${(props) => (props.$active ? props.theme.primary : props.theme.bodyBg)}; - color: ${(props) => props.$active && props.theme.white}; - border: ${(props) => props.theme.gray400} solid 1px; + opacity: ${(props) => props.$isDimmed && 0.33}; + background-color: transparent; ` -const TRANSITION = { - type: 'tween', - ease: 'linear', - duration: 0.2, -} - export interface DatasetSelectorListItemProps { dataset: Dataset isCurrent?: boolean @@ -42,20 +33,13 @@ export interface DatasetSelectorListItemProps { export function DatasetSelectorListItem({ dataset, isCurrent, isDimmed, onClick }: DatasetSelectorListItemProps) { return ( - + ) } -export interface DatasetSelectorListProps extends HTMLProps { +export interface DatasetSelectorListProps { datasets: Dataset[] searchTerm: string datasetHighlighted?: Dataset diff --git a/packages_rs/nextclade-web/yarn.lock b/packages_rs/nextclade-web/yarn.lock index 71cf0a43f..5c15055fe 100644 --- a/packages_rs/nextclade-web/yarn.lock +++ b/packages_rs/nextclade-web/yarn.lock @@ -1998,13 +1998,6 @@ resolved "https://registry.yarnpkg.com/@emotion/hash/-/hash-0.8.0.tgz#bbbff68978fefdbe68ccb533bc8cbe1d1afb5413" integrity sha512-kBJtf7PH6aWwZ6fka3zQ0p6SBYzx4fl1LoZXE2RrnYST9Xljm7WfKJrU4g/Xr3Beg72MLrp1AWNUmuYJTL7Cow== -"@emotion/is-prop-valid@^0.8.2": - version "0.8.8" - resolved "https://registry.yarnpkg.com/@emotion/is-prop-valid/-/is-prop-valid-0.8.8.tgz#db28b1c4368a259b60a97311d6a952d4fd01ac1a" - integrity sha512-u5WtneEAr5IDG2Wv65yhunPSMLIpuKsbuOktRojfrEiEvRyC85LgPMZI63cr7NUqT8ZIGdSVg8ZKGxIug4lXcA== - dependencies: - "@emotion/memoize" "0.7.4" - "@emotion/is-prop-valid@^1.1.0": version "1.1.2" resolved "https://registry.yarnpkg.com/@emotion/is-prop-valid/-/is-prop-valid-1.1.2.tgz#34ad6e98e871aa6f7a20469b602911b8b11b3a95" @@ -2012,11 +2005,6 @@ dependencies: "@emotion/memoize" "^0.7.4" -"@emotion/memoize@0.7.4": - version "0.7.4" - resolved "https://registry.yarnpkg.com/@emotion/memoize/-/memoize-0.7.4.tgz#19bf0f5af19149111c40d98bb0cf82119f5d9eeb" - integrity sha512-Ja/Vfqe3HpuzRsG1oBtWTHk2PGZ7GR+2Vz5iYGelAw8dx32K0y7PjVuxK6z1nMpZOqAFsRUPCkK1YjJ56qJlgw== - "@emotion/memoize@^0.7.4", "@emotion/memoize@^0.7.5": version "0.7.5" resolved "https://registry.yarnpkg.com/@emotion/memoize/-/memoize-0.7.5.tgz#2c40f81449a4e554e9fc6396910ed4843ec2be50" @@ -8021,15 +8009,6 @@ fragment-cache@^0.2.1: dependencies: map-cache "^0.2.2" -framer-motion@10.16.4: - version "10.16.4" - resolved "https://registry.yarnpkg.com/framer-motion/-/framer-motion-10.16.4.tgz#30279ef5499b8d85db3a298ee25c83429933e9f8" - integrity sha512-p9V9nGomS3m6/CALXqv6nFGMuFOxbWsmaOrdmhyQimMIlLl3LC7h7l86wge/Js/8cRu5ktutS/zlzgR7eBOtFA== - dependencies: - tslib "^2.4.0" - optionalDependencies: - "@emotion/is-prop-valid" "^0.8.2" - fresh@0.5.2: version "0.5.2" resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" @@ -15824,11 +15803,6 @@ tslib@^2.0.3, tslib@^2.2.0, tslib@^2.3.1: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.4.0.tgz#7cecaa7f073ce680a05847aa77be941098f36dc3" integrity sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ== -tslib@^2.4.0: - version "2.6.2" - resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" - integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== - tsutils@^3.21.0: version "3.21.0" resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.21.0.tgz#b48717d394cea6c1e096983eed58e9d61715b623" From 5dfb741df4116b160c9239d244ac212740679e08 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Tue, 12 Sep 2023 22:13:05 +0200 Subject: [PATCH 35/51] fix: styled-components warning about updates being too frequent --- .../src/components/Main/DatasetInfo.tsx | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/packages_rs/nextclade-web/src/components/Main/DatasetInfo.tsx b/packages_rs/nextclade-web/src/components/Main/DatasetInfo.tsx index eff4667f9..01ccf4e61 100644 --- a/packages_rs/nextclade-web/src/components/Main/DatasetInfo.tsx +++ b/packages_rs/nextclade-web/src/components/Main/DatasetInfo.tsx @@ -208,18 +208,22 @@ const CountText = styled.span` text-align: center; ` -const CircleBorder = styled.div<{ $percentage: number; $fg?: string; $bg?: string }>` +const CircleBorder = styled.div.attrs((props) => ({ + style: { + background: ` + radial-gradient(closest-side, white 79%, transparent 80% 100%), + conic-gradient( + ${props.$fg ?? props.theme.success} calc(${props.$percentage} * 100%), + ${props.$bg ?? 'lightgray'} 0 + )`, + }, +}))<{ $percentage: number; $fg?: string; $bg?: string }>` display: flex; justify-content: center; align-items: center; border-radius: 50%; width: 75px; height: 75px; - background: radial-gradient(closest-side, white 79%, transparent 80% 100%), - conic-gradient( - ${(props) => props.$fg ?? props.theme.success} calc(${(props) => props.$percentage} * 100%), - ${(props) => props.$bg ?? 'lightgray'} 0 - ); ` const Circle = styled.div<{ $bg?: string; $fg?: string }>` From 07526b8e466c203ae97b8371f3ff65ee0f17341b Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Tue, 12 Sep 2023 22:13:45 +0200 Subject: [PATCH 36/51] feat: make batch flushes more frequent for faster visible response --- .../nextclade-web/src/workers/nextcladeAutodetect.worker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages_rs/nextclade-web/src/workers/nextcladeAutodetect.worker.ts b/packages_rs/nextclade-web/src/workers/nextcladeAutodetect.worker.ts index cc042309e..f2824f216 100644 --- a/packages_rs/nextclade-web/src/workers/nextcladeAutodetect.worker.ts +++ b/packages_rs/nextclade-web/src/workers/nextcladeAutodetect.worker.ts @@ -25,7 +25,7 @@ let nextcladeAutodetect: NextcladeSeqAutodetectWasm | undefined async function create(minimizerIndexJsonStr: MinimizerIndexJson) { nextcladeAutodetect = NextcladeSeqAutodetectWasm.new( JSON.stringify(minimizerIndexJsonStr), - JSON.stringify({ batchIntervalMs: 700, maxBatchSize: 1000 }), + JSON.stringify({ batchIntervalMs: 250, maxBatchSize: 1000 }), ) } From 79a44fc5ced797c63c2e1f638f233708b2ba91d3 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Tue, 12 Sep 2023 22:16:25 +0200 Subject: [PATCH 37/51] fix: typing --- .../nextclade-web/src/components/Main/DatasetInfo.tsx | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages_rs/nextclade-web/src/components/Main/DatasetInfo.tsx b/packages_rs/nextclade-web/src/components/Main/DatasetInfo.tsx index 01ccf4e61..b4bef8841 100644 --- a/packages_rs/nextclade-web/src/components/Main/DatasetInfo.tsx +++ b/packages_rs/nextclade-web/src/components/Main/DatasetInfo.tsx @@ -208,7 +208,13 @@ const CountText = styled.span` text-align: center; ` -const CircleBorder = styled.div.attrs((props) => ({ +interface CircleBorderProps { + $percentage: number + $fg?: string + $bg?: string +} + +const CircleBorder = styled.div.attrs((props) => ({ style: { background: ` radial-gradient(closest-side, white 79%, transparent 80% 100%), @@ -217,7 +223,7 @@ const CircleBorder = styled.div.attrs((props) => ({ ${props.$bg ?? 'lightgray'} 0 )`, }, -}))<{ $percentage: number; $fg?: string; $bg?: string }>` +}))` display: flex; justify-content: center; align-items: center; From f17cf3251116f647b592d621cbbd9333442a7a65 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Tue, 12 Sep 2023 22:24:30 +0200 Subject: [PATCH 38/51] fix: sorting of dataset list --- packages_rs/nextclade-web/src/state/autodetect.state.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages_rs/nextclade-web/src/state/autodetect.state.ts b/packages_rs/nextclade-web/src/state/autodetect.state.ts index 0c3f0b091..ef392c5d1 100644 --- a/packages_rs/nextclade-web/src/state/autodetect.state.ts +++ b/packages_rs/nextclade-web/src/state/autodetect.state.ts @@ -54,7 +54,7 @@ export function groupByDatasets(records: MinimizerSearchRecord[]) { const names = unique(records.flatMap((record) => record.result.datasets.map((dataset) => dataset.name))) let byDataset = {} for (const name of names) { - const selectedRecords = records.find((record) => record.result.datasets.some((dataset) => dataset.name === name)) + const selectedRecords = records.filter((record) => record.result.datasets.some((dataset) => dataset.name === name)) byDataset = { ...byDataset, [name]: selectedRecords } } return byDataset From 407921cadf9aa077dcc4b0a72d4f460850be1850 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Tue, 12 Sep 2023 23:06:34 +0200 Subject: [PATCH 39/51] feat(cli): add dataset statistics into sort command printout --- .../src/cli/nextclade_seq_sort.rs | 46 ++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs index 5d030137b..8ca99c0c1 100644 --- a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs +++ b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs @@ -12,6 +12,7 @@ use nextclade::sort::minimizer_search::{run_minimizer_search, MinimizerSearchRec use nextclade::utils::option::OptionMapRefFallible; use nextclade::utils::string::truncate; use ordered_float::OrderedFloat; +use owo_colors::OwoColorize; use serde::Serialize; use std::collections::btree_map::Entry::{Occupied, Vacant}; use std::collections::BTreeMap; @@ -149,6 +150,8 @@ fn writer_thread( Ok(template) })?; + println!("Suggested datasets for each sequence"); + println!("{}┐", "─".repeat(110)); println!( @@ -159,6 +162,8 @@ fn writer_thread( println!("{}┤", "─".repeat(110)); let mut writers = BTreeMap::new(); + let mut stats = BTreeMap::new(); + let mut n_undetected = 0_usize; for record in result_receiver { let datasets = record @@ -171,11 +176,13 @@ fn writer_thread( print!("{:40}", truncate(&record.fasta_record.seq_name, 40)); if datasets.is_empty() { - println!(" │ {:40} │ {:>10.3} │ {:>10} │", "", "", "",); + println!(" │ {:40} │ {:>10.3} │ {:>10} │", "undetected".red(), "", ""); + n_undetected += 1; } for (i, dataset) in datasets.into_iter().enumerate() { let name = &dataset.name; + *stats.entry(name.clone()).or_insert(1) += 1; let names = name .split('/') @@ -211,6 +218,43 @@ fn writer_thread( println!("{}┤", "─".repeat(110)); } + println!("\n\nSuggested datasets"); + println!("{}┐", "─".repeat(67)); + println!("{:^40} │ {:^10} │ {:^10} │", "Dataset", "Num. seq", "Percent"); + println!("{}┤", "─".repeat(67)); + + let total_seq = stats.values().sum::() + n_undetected; + let stats = stats + .into_iter() + .sorted_by_key(|(name, n_seq)| (-(*n_seq as isize), name.clone())); + for (name, n_seq) in stats { + println!( + "{:<40} │ {:>10} │ {:>9.3}% │", + name, + n_seq, + 100.0 * (n_seq as f64 / total_seq as f64) + ); + } + + if n_undetected > 0 { + println!("{}┤", "─".repeat(67)); + println!( + "{:<40} │ {:>10} │ {:>10} │", + "undetected".red(), + n_undetected.red(), + format!("{:>9.3}%", 100.0 * (n_undetected as f64 / total_seq as f64)).red() + ); + } + + println!("{}┤", "─".repeat(67)); + println!( + "{:>40} │ {:>10} │ {:>10} │", + "total".bold(), + total_seq.bold(), + format!("{:>9.3}%", 100.0).bold() + ); + println!("{}┘", "─".repeat(67)); + Ok(()) } From a81bc88205cf687d485328464cb88d741edbb478 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Wed, 13 Sep 2023 00:29:51 +0200 Subject: [PATCH 40/51] feat(cli): print results of sort command only in verbose mode --- .../src/cli/nextclade_seq_sort.rs | 171 +++++++++++------- 1 file changed, 104 insertions(+), 67 deletions(-) diff --git a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs index 8ca99c0c1..ad1c14cc4 100644 --- a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs +++ b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs @@ -3,7 +3,7 @@ use crate::dataset::dataset_download::download_datasets_index_json; use crate::io::http_client::HttpClient; use eyre::{Report, WrapErr}; use itertools::Itertools; -use log::{info, LevelFilter}; +use log::{trace, LevelFilter}; use nextclade::io::fasta::{FastaReader, FastaRecord, FastaWriter}; use nextclade::io::fs::path_to_string; use nextclade::make_error; @@ -30,7 +30,7 @@ pub fn nextclade_seq_sort(args: &NextcladeSortArgs) -> Result<(), Report> { .. } = args; - let verbose = log::max_level() > LevelFilter::Info; + let verbose = log::max_level() >= LevelFilter::Info; let minimizer_index = if let Some(input_minimizer_index_json) = &input_minimizer_index_json { // If a file is provided, use data from it @@ -64,10 +64,10 @@ pub fn nextclade_seq_sort(args: &NextcladeSortArgs) -> Result<(), Report> { } }?; - run(args, &minimizer_index) + run(args, &minimizer_index, verbose) } -pub fn run(args: &NextcladeSortArgs, minimizer_index: &MinimizerIndexJson) -> Result<(), Report> { +pub fn run(args: &NextcladeSortArgs, minimizer_index: &MinimizerIndexJson, verbose: bool) -> Result<(), Report> { let NextcladeSortArgs { input_fastas, output_dir, @@ -106,7 +106,7 @@ pub fn run(args: &NextcladeSortArgs, minimizer_index: &MinimizerIndexJson) -> Re let result_sender = result_sender.clone(); for fasta_record in &fasta_receiver { - info!("Processing sequence '{}'", fasta_record.seq_name); + trace!("Processing sequence '{}'", fasta_record.seq_name); let result = run_minimizer_search(&fasta_record, minimizer_index, search_params) .wrap_err_with(|| { @@ -130,7 +130,7 @@ pub fn run(args: &NextcladeSortArgs, minimizer_index: &MinimizerIndexJson) -> Re let writer = s.spawn(move || { let output_dir = &output_dir; let output = &output; - writer_thread(output, output_dir, result_receiver).unwrap(); + writer_thread(output, output_dir, result_receiver, verbose).unwrap(); }); }); @@ -141,6 +141,7 @@ fn writer_thread( output: &Option, output_dir: &Option, result_receiver: crossbeam_channel::Receiver, + verbose: bool, ) -> Result<(), Report> { let template = output.map_ref_fallible(move |output| -> Result { let mut template = TinyTemplate::new(); @@ -150,39 +151,14 @@ fn writer_thread( Ok(template) })?; - println!("Suggested datasets for each sequence"); - - println!("{}┐", "─".repeat(110)); - - println!( - "{:^40} │ {:^40} │ {:^10} │ {:^10} │", - "Sequence name", "Dataset", "Score", "Num. hits" - ); - - println!("{}┤", "─".repeat(110)); - let mut writers = BTreeMap::new(); - let mut stats = BTreeMap::new(); - let mut n_undetected = 0_usize; + let mut stats = StatsPrinter::new(verbose); for record in result_receiver { - let datasets = record - .result - .datasets - .iter() - .sorted_by_key(|dataset| -OrderedFloat(dataset.score)) - .collect_vec(); - - print!("{:40}", truncate(&record.fasta_record.seq_name, 40)); - - if datasets.is_empty() { - println!(" │ {:40} │ {:>10.3} │ {:>10} │", "undetected".red(), "", ""); - n_undetected += 1; - } + stats.print_seq(&record); - for (i, dataset) in datasets.into_iter().enumerate() { + for dataset in &record.result.datasets { let name = &dataset.name; - *stats.entry(name.clone()).or_insert(1) += 1; let names = name .split('/') @@ -202,9 +178,64 @@ fn writer_thread( writer.write(&record.fasta_record.seq_name, &record.fasta_record.seq, false)?; } } + } + } + + stats.finish(); + + Ok(()) +} + +struct StatsPrinter { + enabled: bool, + stats: BTreeMap, + n_undetected: usize, +} + +impl StatsPrinter { + pub fn new(enabled: bool) -> Self { + if enabled { + println!("Suggested datasets for each sequence"); + println!("{}┐", "─".repeat(110)); + println!( + "{:^40} │ {:^40} │ {:^10} │ {:^10} │", + "Sequence name", "Dataset", "Score", "Num. hits" + ); + println!("{}┤", "─".repeat(110)); + } + + Self { + enabled, + stats: BTreeMap::new(), + n_undetected: 0, + } + } + + pub fn print_seq(&mut self, record: &MinimizerSearchRecord) { + if !self.enabled { + return; + } + + let datasets = record + .result + .datasets + .iter() + .sorted_by_key(|dataset| -OrderedFloat(dataset.score)) + .collect_vec(); + + print!("{:<40}", truncate(&record.fasta_record.seq_name, 40)); + + if datasets.is_empty() { + println!(" │ {:40} │ {:>10.3} │ {:>10} │", "undetected".red(), "", ""); + self.n_undetected += 1; + } + + for (i, dataset) in datasets.into_iter().enumerate() { + let name = &dataset.name; + *self.stats.entry(name.clone()).or_insert(1) += 1; if i != 0 { - print!("{:40}", ""); + print!("{:<40}", ""); } println!( @@ -218,44 +249,50 @@ fn writer_thread( println!("{}┤", "─".repeat(110)); } - println!("\n\nSuggested datasets"); - println!("{}┐", "─".repeat(67)); - println!("{:^40} │ {:^10} │ {:^10} │", "Dataset", "Num. seq", "Percent"); - println!("{}┤", "─".repeat(67)); + pub fn finish(&self) { + if !self.enabled { + return; + } - let total_seq = stats.values().sum::() + n_undetected; - let stats = stats - .into_iter() - .sorted_by_key(|(name, n_seq)| (-(*n_seq as isize), name.clone())); - for (name, n_seq) in stats { - println!( - "{:<40} │ {:>10} │ {:>9.3}% │", - name, - n_seq, - 100.0 * (n_seq as f64 / total_seq as f64) - ); - } + println!("\n\nSuggested datasets"); + println!("{}┐", "─".repeat(67)); + println!("{:^40} │ {:^10} │ {:^10} │", "Dataset", "Num. seq", "Percent"); + println!("{}┤", "─".repeat(67)); + + let total_seq = self.stats.values().sum::() + self.n_undetected; + let stats = self + .stats + .iter() + .sorted_by_key(|(name, n_seq)| (-(**n_seq as isize), (*name).clone())); + + for (name, n_seq) in stats { + println!( + "{:<40} │ {:>10} │ {:>9.3}% │", + name, + n_seq, + 100.0 * (*n_seq as f64 / total_seq as f64) + ); + } + + if self.n_undetected > 0 { + println!("{}┤", "─".repeat(67)); + println!( + "{:<40} │ {:>10} │ {:>10} │", + "undetected".red(), + self.n_undetected.red(), + format!("{:>9.3}%", 100.0 * (self.n_undetected as f64 / total_seq as f64)).red() + ); + } - if n_undetected > 0 { println!("{}┤", "─".repeat(67)); println!( - "{:<40} │ {:>10} │ {:>10} │", - "undetected".red(), - n_undetected.red(), - format!("{:>9.3}%", 100.0 * (n_undetected as f64 / total_seq as f64)).red() + "{:>40} │ {:>10} │ {:>10} │", + "total".bold(), + total_seq.bold(), + format!("{:>9.3}%", 100.0).bold() ); + println!("{}┘", "─".repeat(67)); } - - println!("{}┤", "─".repeat(67)); - println!( - "{:>40} │ {:>10} │ {:>10} │", - "total".bold(), - total_seq.bold(), - format!("{:>9.3}%", 100.0).bold() - ); - println!("{}┘", "─".repeat(67)); - - Ok(()) } fn get_or_insert_writer( From d4efb6251337331dc2c6c069a4fee926f1af3163 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Wed, 13 Sep 2023 01:34:22 +0200 Subject: [PATCH 41/51] feat(cli): add output tsv file for sort command --- .../nextclade-cli/src/cli/nextclade_cli.rs | 22 +++---- .../src/cli/nextclade_seq_sort.rs | 57 ++++++++++++++----- packages_rs/nextclade/src/utils/option.rs | 23 ++++++++ 3 files changed, 77 insertions(+), 25 deletions(-) diff --git a/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs b/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs index 1ace36843..710c6c218 100644 --- a/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs +++ b/packages_rs/nextclade-cli/src/cli/nextclade_cli.rs @@ -638,7 +638,6 @@ pub struct NextcladeSortArgs { /// /// See: https://en.wikipedia.org/wiki/FASTA_format #[clap(value_hint = ValueHint::FilePath)] - #[clap(display_order = 1)] pub input_fastas: Vec, /// Path to input minimizer index JSON file. @@ -648,7 +647,6 @@ pub struct NextcladeSortArgs { /// Supports the following compression formats: "gz", "bz2", "xz", "zst". Use "-" to read uncompressed data from standard input (stdin). #[clap(long, short = 'm')] #[clap(value_hint = ValueHint::FilePath)] - #[clap(display_order = 1)] pub input_minimizer_index_json: Option, /// Path to output directory @@ -657,9 +655,8 @@ pub struct NextcladeSortArgs { /// /// Mutually exclusive with `--output`. /// - #[clap(long)] + #[clap(short = 'O', long)] #[clap(value_hint = ValueHint::DirPath)] - #[clap(hide_long_help = true, hide_short_help = true)] #[clap(group = "outputs")] pub output_dir: Option, @@ -671,18 +668,21 @@ pub struct NextcladeSortArgs { /// /// Mutually exclusive with `--output-dir`. /// - /// If the provided file path ends with one of the supported extensions: "gz", "bz2", "xz", "zst", then the file will be written compressed. - /// - /// If the required directory tree does not exist, it will be created. + /// If the provided file path ends with one of the supported extensions: "gz", "bz2", "xz", "zst", then the file will be written compressed. If the required directory tree does not exist, it will be created. /// /// Example for bash shell: /// /// --output='outputs/{name}/sorted.fasta.gz' - #[clap(long)] - #[clap(value_hint = ValueHint::DirPath)] - #[clap(hide_long_help = true, hide_short_help = true)] + #[clap(short = 'o', long)] #[clap(group = "outputs")] - pub output: Option, + pub output_path: Option, + + /// Path to output results TSV file + /// + /// If the provided file path ends with one of the supported extensions: "gz", "bz2", "xz", "zst", then the file will be written compressed. Use "-" to write uncompressed to standard output (stdout). If the required directory tree does not exist, it will be created. + #[clap(short = 'r', long)] + #[clap(value_hint = ValueHint::FilePath)] + pub output_results_tsv: Option, #[clap(flatten, next_help_heading = "Algorithm")] pub search_params: NextcladeSeqSortParams, diff --git a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs index ad1c14cc4..a00bb7fbb 100644 --- a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs +++ b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs @@ -4,15 +4,17 @@ use crate::io::http_client::HttpClient; use eyre::{Report, WrapErr}; use itertools::Itertools; use log::{trace, LevelFilter}; +use nextclade::io::csv::CsvStructFileWriter; use nextclade::io::fasta::{FastaReader, FastaRecord, FastaWriter}; use nextclade::io::fs::path_to_string; use nextclade::make_error; use nextclade::sort::minimizer_index::{MinimizerIndexJson, MINIMIZER_INDEX_ALGO_VERSION}; use nextclade::sort::minimizer_search::{run_minimizer_search, MinimizerSearchRecord}; -use nextclade::utils::option::OptionMapRefFallible; +use nextclade::utils::option::{OptionMapMutFallible, OptionMapRefFallible}; use nextclade::utils::string::truncate; use ordered_float::OrderedFloat; use owo_colors::OwoColorize; +use schemars::JsonSchema; use serde::Serialize; use std::collections::btree_map::Entry::{Occupied, Vacant}; use std::collections::BTreeMap; @@ -55,12 +57,12 @@ pub fn nextclade_seq_sort(args: &NextcladeSortArgs) -> Result<(), Report> { .map(|minimizer_index| format!("'{}'", minimizer_index.version)) .join(","); let server_versions = if server_versions.is_empty() { - "none".to_owned() + "none available".to_owned() } else { format!(": {server_versions}") }; - make_error!("No compatible reference minimizer index data is found for this dataset sever. Cannot proceed. \n\nThis version of Nextclade supports index versions up to '{}', but the server has{}.\n\nTry to to upgrade Nextclade to the latest version and/or contact dataset server maintainers.", MINIMIZER_INDEX_ALGO_VERSION, server_versions) + make_error!("No compatible reference minimizer index data is found for this dataset sever. Cannot proceed. \n\nThis version of Nextclade supports index versions up to '{}', but the server has {}.\n\nTry to to upgrade Nextclade to the latest version and/or contact dataset server maintainers.", MINIMIZER_INDEX_ALGO_VERSION, server_versions) } }?; @@ -70,8 +72,6 @@ pub fn nextclade_seq_sort(args: &NextcladeSortArgs) -> Result<(), Report> { pub fn run(args: &NextcladeSortArgs, minimizer_index: &MinimizerIndexJson, verbose: bool) -> Result<(), Report> { let NextcladeSortArgs { input_fastas, - output_dir, - output, search_params, other_params: NextcladeRunOtherParams { jobs }, .. @@ -128,38 +128,63 @@ pub fn run(args: &NextcladeSortArgs, minimizer_index: &MinimizerIndexJson, verbo } let writer = s.spawn(move || { - let output_dir = &output_dir; - let output = &output; - writer_thread(output, output_dir, result_receiver, verbose).unwrap(); + writer_thread(args, result_receiver, verbose).unwrap(); }); }); Ok(()) } +#[derive(Clone, Default, Debug, Serialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +struct SeqSortCsvEntry<'a> { + seq_name: &'a str, + dataset: &'a str, + score: f64, + num_hits: u64, +} + fn writer_thread( - output: &Option, - output_dir: &Option, + args: &NextcladeSortArgs, result_receiver: crossbeam_channel::Receiver, verbose: bool, ) -> Result<(), Report> { - let template = output.map_ref_fallible(move |output| -> Result { + let NextcladeSortArgs { + output_dir, + output_path, + output_results_tsv, + .. + } = args; + + let template = output_path.map_ref_fallible(move |output_path| -> Result { let mut template = TinyTemplate::new(); template - .add_template("output", output) - .wrap_err_with(|| format!("When parsing template: {output}"))?; + .add_template("output", output_path) + .wrap_err_with(|| format!("When parsing template: '{output_path}'"))?; Ok(template) })?; let mut writers = BTreeMap::new(); let mut stats = StatsPrinter::new(verbose); + let mut results_csv = + output_results_tsv.map_ref_fallible(|output_results_tsv| CsvStructFileWriter::new(output_results_tsv, b'\t'))?; + for record in result_receiver { stats.print_seq(&record); for dataset in &record.result.datasets { let name = &dataset.name; + results_csv.map_mut_fallible(|results_csv| { + results_csv.write(&SeqSortCsvEntry { + seq_name: &record.fasta_record.seq_name, + dataset: &dataset.name, + score: dataset.score, + num_hits: dataset.n_hits, + }) + })?; + let names = name .split('/') .scan(PathBuf::new(), |name, component| { @@ -329,7 +354,11 @@ struct OutputTemplateContext<'a> { } fn check_args(args: &NextcladeSortArgs) -> Result<(), Report> { - let NextcladeSortArgs { output_dir, output, .. } = args; + let NextcladeSortArgs { + output_dir, + output_path: output, + .. + } = args; if output.is_some() && output_dir.is_some() { return make_error!( diff --git a/packages_rs/nextclade/src/utils/option.rs b/packages_rs/nextclade/src/utils/option.rs index cb0ad0f5a..5ff800ad2 100644 --- a/packages_rs/nextclade/src/utils/option.rs +++ b/packages_rs/nextclade/src/utils/option.rs @@ -36,3 +36,26 @@ impl<'o, T: 'o> OptionMapRefFallible<'o, T> for Option { (*self).as_ref().map(f).transpose() } } + +pub trait OptionMapMutFallible<'o, T: 'o> { + /// Borrows the internal value of an `Option`, maps it using the provided closure + /// and transposes `Option` of `Result` to `Result` of `Option`. + /// + /// Convenient to use with fallible mapping functions (which returns a `Result`) + /// + /// Inspired by + /// https://github.com/ammongit/rust-ref-map/blob/4b1251c6d2fd192d89a114395b36aeeab5c5433c/src/option.rs + fn map_mut_fallible(&'o mut self, f: F) -> Result, E> + where + F: FnOnce(&'o mut T) -> Result; +} + +impl<'o, T: 'o> OptionMapMutFallible<'o, T> for Option { + #[inline] + fn map_mut_fallible(&'o mut self, f: F) -> Result, E> + where + F: FnOnce(&'o mut T) -> Result, + { + (*self).as_mut().map(f).transpose() + } +} From 99fb0c89ac33fe22ddf12f6d4c5b4ff10e81eac5 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Wed, 13 Sep 2023 01:38:16 +0200 Subject: [PATCH 42/51] fix(cli): add undetected entries into sort tsv --- .../src/cli/nextclade_seq_sort.rs | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs index a00bb7fbb..847a3d1c4 100644 --- a/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs +++ b/packages_rs/nextclade-cli/src/cli/nextclade_seq_sort.rs @@ -139,9 +139,9 @@ pub fn run(args: &NextcladeSortArgs, minimizer_index: &MinimizerIndexJson, verbo #[serde(rename_all = "camelCase")] struct SeqSortCsvEntry<'a> { seq_name: &'a str, - dataset: &'a str, - score: f64, - num_hits: u64, + dataset: Option<&'a str>, + score: Option, + num_hits: Option, } fn writer_thread( @@ -173,15 +173,28 @@ fn writer_thread( for record in result_receiver { stats.print_seq(&record); - for dataset in &record.result.datasets { + let datasets = &record.result.datasets; + + if datasets.is_empty() { + results_csv.map_mut_fallible(|results_csv| { + results_csv.write(&SeqSortCsvEntry { + seq_name: &record.fasta_record.seq_name, + dataset: None, + score: None, + num_hits: None, + }) + })?; + } + + for dataset in datasets { let name = &dataset.name; results_csv.map_mut_fallible(|results_csv| { results_csv.write(&SeqSortCsvEntry { seq_name: &record.fasta_record.seq_name, - dataset: &dataset.name, - score: dataset.score, - num_hits: dataset.n_hits, + dataset: Some(&dataset.name), + score: Some(dataset.score), + num_hits: Some(dataset.n_hits), }) })?; From 131fedd85891d46b6205d09c5c954df97fbf8bd1 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Thu, 14 Sep 2023 08:10:33 +0200 Subject: [PATCH 43/51] feat(web): reimplement main page --- .../src/components/Common/List.tsx | 34 ++- .../src/components/Common/SearchBox.tsx | 96 +++++++++ .../src/components/FilePicker/FilePicker.tsx | 1 + .../src/components/FilePicker/UploadBox.tsx | 1 + .../src/components/Main/DatasetInfo.tsx | 25 ++- .../src/components/Main/DatasetSelector.tsx | 198 +++++++----------- .../components/Main/DatasetSelectorList.tsx | 123 ++++++----- .../src/components/Main/MainInputForm.tsx | 7 +- .../Main/QuerySequenceFilePicker.tsx | 155 +++++--------- .../src/components/Main/QuerySequenceList.tsx | 145 ++++++------- .../src/components/Main/RunPanel.tsx | 132 ++++++++++++ .../src/components/Main/SuggestionPanel.tsx | 101 +++++++++ .../nextclade-web/src/state/dataset.state.ts | 2 - .../nextclade-web/src/state/inputs.state.ts | 9 +- 14 files changed, 650 insertions(+), 379 deletions(-) create mode 100644 packages_rs/nextclade-web/src/components/Common/SearchBox.tsx create mode 100644 packages_rs/nextclade-web/src/components/Main/RunPanel.tsx create mode 100644 packages_rs/nextclade-web/src/components/Main/SuggestionPanel.tsx diff --git a/packages_rs/nextclade-web/src/components/Common/List.tsx b/packages_rs/nextclade-web/src/components/Common/List.tsx index e8496a60f..0ec0fdc27 100644 --- a/packages_rs/nextclade-web/src/components/Common/List.tsx +++ b/packages_rs/nextclade-web/src/components/Common/List.tsx @@ -1,4 +1,4 @@ -import styled from 'styled-components' +import styled, { css } from 'styled-components' export const Ul = styled.ul` padding-left: 1.5rem; @@ -13,3 +13,35 @@ export const UlInvisible = styled.ul` export const LiInvisible = styled.li` list-style: none; ` + +// @formatter:off +// prettier-ignore +export const ScrollShadowVerticalCss = css` + /** Taken from: https://css-tricks.com/books/greatest-css-tricks/scroll-shadows */ + background: + /* Shadow Cover TOP */ linear-gradient(white 30%, rgba(255, 255, 255, 0)) center top, + /* Shadow Cover BOTTOM */ linear-gradient(rgba(255, 255, 255, 0), white 70%) center bottom, + /* Shadow TOP */ radial-gradient(farthest-side at 50% 0, rgba(0, 0, 0, 0.2), rgba(0, 0, 0, 0)) center top, + /* Shadow BOTTOM */ radial-gradient(farthest-side at 50% 100%, rgba(0, 0, 0, 0.2), rgba(0, 0, 0, 0)) center bottom; + background-repeat: no-repeat; + background-size: 100% 40px, 100% 40px, 100% 14px, 100% 14px; + background-attachment: local, local, scroll, scroll; +` +// @formatter:on + +export const ListGenericCss = css` + ${ScrollShadowVerticalCss}; + list-style: none; + padding: 0; + margin: 0; + -webkit-overflow-scrolling: touch; + overflow-scrolling: touch; + + & li { + border: 0; + } +` + +export const UlGeneric = styled.ul` + ${ListGenericCss} +` diff --git a/packages_rs/nextclade-web/src/components/Common/SearchBox.tsx b/packages_rs/nextclade-web/src/components/Common/SearchBox.tsx new file mode 100644 index 000000000..c5b634d7a --- /dev/null +++ b/packages_rs/nextclade-web/src/components/Common/SearchBox.tsx @@ -0,0 +1,96 @@ +import React, { ChangeEvent, useCallback, useMemo, HTMLProps } from 'react' +import styled from 'styled-components' +import { Form, Input as InputBase } from 'reactstrap' +import { MdSearch as IconSearchBase, MdClear as IconClearBase } from 'react-icons/md' +import { useTranslationSafe } from 'src/helpers/useTranslationSafe' +import { ButtonTransparent } from 'src/components/Common/ButtonTransparent' + +const SearchForm = styled(Form)` + display: inline; + position: relative; +` + +const IconSearchWrapper = styled.span` + display: inline; + position: absolute; + padding: 5px 7px; +` + +const IconSearch = styled(IconSearchBase)` + * { + color: ${(props) => props.theme.gray500}; + } +` + +const ButtonClear = styled(ButtonTransparent)` + display: inline; + position: absolute; + right: 0; + padding: 0 7px; +` + +const IconClear = styled(IconClearBase)` + * { + color: ${(props) => props.theme.gray500}; + } +` + +const Input = styled(InputBase)` + display: inline !important; + padding-left: 35px; + padding-right: 30px; + height: 2.2em; +` + +export interface SearchBoxProps extends Omit, 'as'> { + searchTitle?: string + searchTerm: string + onSearchTermChange(term: string): void +} + +export function SearchBox({ searchTitle, searchTerm, onSearchTermChange, ...restProps }: SearchBoxProps) { + const { t } = useTranslationSafe() + + const onChange = useCallback( + (event: ChangeEvent) => { + onSearchTermChange(event.target.value) + }, + [onSearchTermChange], + ) + + const onClear = useCallback(() => { + onSearchTermChange('') + }, [onSearchTermChange]) + + const buttonClear = useMemo(() => { + if (searchTerm.length === 0) { + return null + } + return ( + + + + ) + }, [onClear, searchTerm.length, t]) + + return ( + + + + + + {buttonClear} + + ) +} diff --git a/packages_rs/nextclade-web/src/components/FilePicker/FilePicker.tsx b/packages_rs/nextclade-web/src/components/FilePicker/FilePicker.tsx index 30f2e95b9..cdb00d85c 100644 --- a/packages_rs/nextclade-web/src/components/FilePicker/FilePicker.tsx +++ b/packages_rs/nextclade-web/src/components/FilePicker/FilePicker.tsx @@ -16,6 +16,7 @@ import { UploadedFileInfo } from './UploadedFileInfo' import { UploadedFileInfoCompact } from './UploadedFileInfoCompact' export const FilePickerContainer = styled.div` + flex: 1; display: flex; flex-direction: column; ` diff --git a/packages_rs/nextclade-web/src/components/FilePicker/UploadBox.tsx b/packages_rs/nextclade-web/src/components/FilePicker/UploadBox.tsx index 24d1995e6..d559604e2 100644 --- a/packages_rs/nextclade-web/src/components/FilePicker/UploadBox.tsx +++ b/packages_rs/nextclade-web/src/components/FilePicker/UploadBox.tsx @@ -84,6 +84,7 @@ export function UploadBox({ onUpload, children, multiple = false, ...props }: Pr () => ( {t('Drag & drop files')} + {t('or folders')} {t('Select files')} ), diff --git a/packages_rs/nextclade-web/src/components/Main/DatasetInfo.tsx b/packages_rs/nextclade-web/src/components/Main/DatasetInfo.tsx index b4bef8841..db8e79afc 100644 --- a/packages_rs/nextclade-web/src/components/Main/DatasetInfo.tsx +++ b/packages_rs/nextclade-web/src/components/Main/DatasetInfo.tsx @@ -15,9 +15,21 @@ import { import type { Dataset } from 'src/types' import styled from 'styled-components' -export const DatasetInfoContainer = styled.div` +export const Container = styled.div` display: flex; - flex-direction: row; + //border: 1px #ccc9 solid; + //border-radius: 5px; + + //margin-top: 3px !important; + //margin-bottom: 3px !important; + //margin-left: 5px; + //padding: 15px; + + margin: 0; + padding: 15px; + box-shadow: 0 0 12px 0 #0002; + border: 1px #ccc9 solid; + border-radius: 5px; ` export const FlexLeft = styled.div` @@ -85,7 +97,7 @@ export function DatasetInfo({ dataset }: DatasetInfoProps) { } return ( - + @@ -144,7 +156,7 @@ export function DatasetInfo({ dataset }: DatasetInfoProps) { {t('Updated at: {{updated}}', { updated: updatedAt })} {t('Dataset name: {{name}}', { name: path })} - + ) } @@ -152,14 +164,14 @@ export function DatasetUndetectedInfo() { const { t } = useTranslationSafe() return ( - + {t('Autodetect')} {t('Detect pathogen automatically from sequences')} - + ) } @@ -206,6 +218,7 @@ function DatasetInfoAutodetectProgressCircle({ dataset }: DatasetInfoCircleProps const CountText = styled.span` text-align: center; + font-size: 0.8rem; ` interface CircleBorderProps { diff --git a/packages_rs/nextclade-web/src/components/Main/DatasetSelector.tsx b/packages_rs/nextclade-web/src/components/Main/DatasetSelector.tsx index 2fb2eae9b..ca7f290b4 100644 --- a/packages_rs/nextclade-web/src/components/Main/DatasetSelector.tsx +++ b/packages_rs/nextclade-web/src/components/Main/DatasetSelector.tsx @@ -1,38 +1,89 @@ -import { isNil } from 'lodash' -import React, { HTMLProps, useCallback, useState } from 'react' +import React, { HTMLProps, useState } from 'react' +import { useRecoilState, useRecoilValue } from 'recoil' +import styled from 'styled-components' import { ThreeDots } from 'react-loader-spinner' -import { Button, Col, Container, Form, FormGroup, Input, Row } from 'reactstrap' -import { useRecoilState, useRecoilValue, useResetRecoilState } from 'recoil' -import { Toggle } from 'src/components/Common/Toggle' +import { SuggestionPanel } from 'src/components/Main/SuggestionPanel' import { useTranslationSafe } from 'src/helpers/useTranslationSafe' -import { useRecoilToggle } from 'src/hooks/useToggle' -import { autodetectResultsAtom, hasAutodetectResultsAtom } from 'src/state/autodetect.state' -import { datasetCurrentAtom, datasetsAtom, minimizerIndexVersionAtom } from 'src/state/dataset.state' -import { shouldSuggestDatasetsAtom } from 'src/state/settings.state' -import styled from 'styled-components' -import { DatasetSelectorList } from './DatasetSelectorList' +import { datasetCurrentAtom, datasetsAtom } from 'src/state/dataset.state' +import { SearchBox } from 'src/components/Common/SearchBox' +import { DatasetSelectorList } from 'src/components/Main/DatasetSelectorList' + +export function DatasetSelector() { + const { t } = useTranslationSafe() + const [searchTerm, setSearchTerm] = useState('') + const { datasets } = useRecoilValue(datasetsAtom) + const [datasetCurrent, setDatasetCurrent] = useRecoilState(datasetCurrentAtom) + + const isBusy = datasets.length === 0 + + return ( + +
+ {t('Select dataset')} + + +
+ +
+ {!isBusy && ( + + )} + + {isBusy && ( + + + + + + )} +
+ +
+ +
+ + ) +} -const DatasetSelectorContainer = styled(Container)` +const Container = styled.div` display: flex; + flex: 1; flex-direction: column; - width: 100%; height: 100%; overflow: hidden; - padding: 0; + margin-right: 10px; ` -const DatasetSelectorTitle = styled.h4` - flex: 1; - margin: auto 0; +const Header = styled.div` + display: flex; + flex: 0; + padding-left: 10px; + margin-top: 10px; + margin-bottom: 3px; ` -const DatasetSelectorListContainer = styled.section` +const Main = styled.div` display: flex; - width: 100%; - height: 100%; + flex: 1; + flex-direction: column; overflow: hidden; ` +const Footer = styled.div` + display: flex; + flex: 0; +` + +const Title = styled.h4` + flex: 1; + margin: auto 0; +` + const SpinnerWrapper = styled.div>` width: 100%; height: 100%; @@ -48,110 +99,3 @@ const Spinner = styled(ThreeDots)` margin: auto; height: 100%; ` - -export function DatasetSelector() { - const { t } = useTranslationSafe() - const [searchTerm, setSearchTerm] = useState('') - const { datasets } = useRecoilValue(datasetsAtom) - const [datasetCurrent, setDatasetCurrent] = useRecoilState(datasetCurrentAtom) - - const onSearchTermChange = useCallback( - (event: React.ChangeEvent) => { - const { value } = event.target - setSearchTerm(value) - }, - [setSearchTerm], - ) - - const isBusy = datasets.length === 0 - - return ( - - - - {t('Select pathogen dataset')} - - - - - - - - - - - - - - - - - {!isBusy && ( - - )} - - {isBusy && ( - - - - - - )} - - - - - ) -} - -function AutodetectToggle() { - const { t } = useTranslationSafe() - const minimizerIndexVersion = useRecoilValue(minimizerIndexVersionAtom) - const resetAutodetectResults = useResetRecoilState(autodetectResultsAtom) - const hasAutodetectResults = useRecoilValue(hasAutodetectResultsAtom) - const { state: shouldSuggestDatasets, toggle: toggleSuggestDatasets } = useRecoilToggle(shouldSuggestDatasetsAtom) - - if (isNil(minimizerIndexVersion)) { - return null - } - - return ( - - - - - {t('Suggest best matches')} - - - - - - - ) -} diff --git a/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx b/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx index a5c69c2d9..d02a23bde 100644 --- a/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx +++ b/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx @@ -1,7 +1,9 @@ import { get, isNil, sortBy } from 'lodash' +import { lighten } from 'polished' import React, { useCallback, useMemo } from 'react' import { ListGroup, ListGroupItem } from 'reactstrap' import { useRecoilValue } from 'recoil' +import { ListGenericCss } from 'src/components/Common/List' import styled from 'styled-components' import type { Dataset } from 'src/types' import { areDatasetsEqual } from 'src/types' @@ -9,36 +11,6 @@ import { autodetectResultsAtom, groupByDatasets } from 'src/state/autodetect.sta import { search } from 'src/helpers/search' import { DatasetInfo } from 'src/components/Main/DatasetInfo' -export const DatasetSelectorUl = styled(ListGroup)` - flex: 1; - overflow-y: scroll; - height: 100%; -` - -export const DatasetSelectorLi = styled(ListGroupItem)<{ $isDimmed?: boolean }>` - list-style: none; - margin: 0; - padding: 0.5rem; - cursor: pointer; - opacity: ${(props) => props.$isDimmed && 0.33}; - background-color: transparent; -` - -export interface DatasetSelectorListItemProps { - dataset: Dataset - isCurrent?: boolean - isDimmed?: boolean - onClick?: () => void -} - -export function DatasetSelectorListItem({ dataset, isCurrent, isDimmed, onClick }: DatasetSelectorListItemProps) { - return ( - - - - ) -} - export interface DatasetSelectorListProps { datasets: Dataset[] searchTerm: string @@ -93,30 +65,73 @@ export function DatasetSelectorList({ const { itemsStartWith, itemsInclude, itemsNotInclude } = searchResult + const listItems = useMemo(() => { + return ( + <> + {[itemsStartWith, itemsInclude].map((datasets) => + datasets.map((dataset) => ( + + )), + )} + + {[itemsNotInclude].map((datasets) => + datasets.map((dataset) => ( + + )), + )} + + ) + }, [datasetHighlighted, itemsInclude, itemsNotInclude, itemsStartWith, onItemClick]) + + return
    {listItems}
+} + +export const Ul = styled(ListGroup)` + ${ListGenericCss}; + flex: 1; + overflow: auto; + padding: 5px 5px; + border-radius: 0 !important; +` + +export const Li = styled(ListGroupItem)<{ $isDimmed?: boolean }>` + cursor: pointer; + opacity: ${(props) => props.$isDimmed && 0.4}; + background-color: transparent; + + margin: 3px 3px !important; + padding: 0 !important; + border-radius: 5px !important; + + &.active { + background-color: ${(props) => lighten(0.033)(props.theme.primary)}; + box-shadow: -3px 3px 12px 3px #0005; + opacity: ${(props) => props.$isDimmed && 0.66}; + } +` + +interface DatasetSelectorListItemProps { + dataset: Dataset + isCurrent?: boolean + isDimmed?: boolean + onClick?: () => void +} + +function DatasetSelectorListItem({ dataset, isCurrent, isDimmed, onClick }: DatasetSelectorListItemProps) { return ( - - {[itemsStartWith, itemsInclude].map((datasets) => - datasets.map((dataset) => ( - - )), - )} - - {[itemsNotInclude].map((datasets) => - datasets.map((dataset) => ( - - )), - )} - +
  • + +
    • ) } diff --git a/packages_rs/nextclade-web/src/components/Main/MainInputForm.tsx b/packages_rs/nextclade-web/src/components/Main/MainInputForm.tsx index c3c0a890b..94b4b9e84 100644 --- a/packages_rs/nextclade-web/src/components/Main/MainInputForm.tsx +++ b/packages_rs/nextclade-web/src/components/Main/MainInputForm.tsx @@ -2,12 +2,13 @@ import React from 'react' import { QuerySequenceFilePicker } from 'src/components/Main/QuerySequenceFilePicker' import styled from 'styled-components' import { Col as ColBase, Row as RowBase } from 'reactstrap' -import { DatasetSelector } from 'src/components/Main/DatasetSelector' import { useUpdatedDatasetIndex } from 'src/io/fetchDatasets' +import { DatasetSelector } from 'src/components/Main/DatasetSelector' const Container = styled.div` height: 100%; overflow: hidden; + margin-top: 10px; ` const Row = styled(RowBase)` @@ -27,10 +28,10 @@ export function MainInputForm() { return ( - + - + diff --git a/packages_rs/nextclade-web/src/components/Main/QuerySequenceFilePicker.tsx b/packages_rs/nextclade-web/src/components/Main/QuerySequenceFilePicker.tsx index 8eed4018f..c0f3e4bc5 100644 --- a/packages_rs/nextclade-web/src/components/Main/QuerySequenceFilePicker.tsx +++ b/packages_rs/nextclade-web/src/components/Main/QuerySequenceFilePicker.tsx @@ -1,51 +1,28 @@ import React, { useCallback, useMemo } from 'react' -import { Button, Col, Form, FormGroup, Row } from 'reactstrap' import { useRecoilValue } from 'recoil' +import styled from 'styled-components' +import type { AlgorithmInput } from 'src/types' import { QuerySequenceList } from 'src/components/Main/QuerySequenceList' +import { RunPanel } from 'src/components/Main/RunPanel' import { useRunAnalysis } from 'src/hooks/useRunAnalysis' import { useRunSeqAutodetect } from 'src/hooks/useRunSeqAutodetect' import { useRecoilToggle } from 'src/hooks/useToggle' -import { canRunAtom } from 'src/state/results.state' -import styled from 'styled-components' -import { datasetCurrentAtom } from 'src/state/dataset.state' -import { hasInputErrorsAtom, qrySeqErrorAtom } from 'src/state/error.state' +import { qrySeqErrorAtom } from 'src/state/error.state' import { shouldRunAutomaticallyAtom, shouldSuggestDatasetsAtom } from 'src/state/settings.state' -import type { AlgorithmInput } from 'src/types' -import { Toggle } from 'src/components/Common/Toggle' -import { FlexLeft, FlexRight } from 'src/components/FilePicker/FilePickerStyles' import { useTranslationSafe } from 'src/helpers/useTranslationSafe' -import { AlgorithmInputDefault } from 'src/io/AlgorithmInput' import { FilePicker } from 'src/components/FilePicker/FilePicker' import { FileIconFasta } from 'src/components/Common/FileIcons' -import { hasRequiredInputsAtom, useQuerySeqInputs } from 'src/state/inputs.state' - -const SequenceFilePickerContainer = styled.section` - display: flex; - flex-direction: column; - width: 100%; - height: 100%; -` - -const ButtonRunStyled = styled(Button)` - min-width: 160px; - min-height: 50px; - margin-left: 1rem; -` +import { useQuerySeqInputs } from 'src/state/inputs.state' export function QuerySequenceFilePicker() { const { t } = useTranslationSafe() - const datasetCurrent = useRecoilValue(datasetCurrentAtom) const { qryInputs, addQryInputs } = useQuerySeqInputs() const qrySeqError = useRecoilValue(qrySeqErrorAtom) - const canRun = useRecoilValue(canRunAtom) - const { state: shouldRunAutomatically, toggle: toggleRunAutomatically } = useRecoilToggle(shouldRunAutomaticallyAtom) + const { state: shouldRunAutomatically } = useRecoilToggle(shouldRunAutomaticallyAtom) const shouldSuggestDatasets = useRecoilValue(shouldSuggestDatasetsAtom) - const hasRequiredInputs = useRecoilValue(hasRequiredInputsAtom) - const hasInputErrors = useRecoilValue(hasInputErrorsAtom) - const icon = useMemo(() => , []) const runAnalysis = useRunAnalysis() @@ -64,29 +41,6 @@ export function QuerySequenceFilePicker() { [addQryInputs, runAnalysis, runAutodetect, shouldRunAutomatically, shouldSuggestDatasets], ) - const setExampleSequences = useCallback(() => { - if (datasetCurrent) { - addQryInputs([new AlgorithmInputDefault(datasetCurrent)]) - if (shouldSuggestDatasets) { - runAutodetect() - } - if (shouldRunAutomatically) { - runAnalysis() - } - } - }, [addQryInputs, datasetCurrent, runAnalysis, runAutodetect, shouldRunAutomatically, shouldSuggestDatasets]) - - const { isRunButtonDisabled, runButtonColor, runButtonTooltip } = useMemo(() => { - const isRunButtonDisabled = !(canRun && hasRequiredInputs) || hasInputErrors - return { - isRunButtonDisabled, - runButtonColor: isRunButtonDisabled ? 'secondary' : 'success', - runButtonTooltip: isRunButtonDisabled - ? t('Please provide input files for the algorithm') - : t('Launch the algorithm!'), - } - }, [canRun, hasInputErrors, hasRequiredInputs, t]) - const headerText = useMemo(() => { if (qryInputs.length > 0) { return t('Add more sequence data') @@ -95,56 +49,55 @@ export function QuerySequenceFilePicker() { }, [qryInputs.length, t]) return ( - - + +
    + +
    + +
    + +
    + +
    + +
    +     + ) +} - +const Container = styled.div` + display: flex; + flex: 1; + flex-direction: column; + height: 100%; + overflow: hidden; + margin-left: 10px; +` - - - -
    - - - - {t('Run automatically')} - - - -
    -     +const Header = styled.div` + display: flex; + flex: 0; + margin-bottom: 15px; +` - - +const Main = styled.div` + display: flex; + flex: 1; + flex-direction: column; + overflow: hidden; +` - - {t('Run')} - - - -     -     - ) -} +const Footer = styled.div` + display: flex; + flex: 0; +` diff --git a/packages_rs/nextclade-web/src/components/Main/QuerySequenceList.tsx b/packages_rs/nextclade-web/src/components/Main/QuerySequenceList.tsx index d7ec69ab5..cb7802f94 100644 --- a/packages_rs/nextclade-web/src/components/Main/QuerySequenceList.tsx +++ b/packages_rs/nextclade-web/src/components/Main/QuerySequenceList.tsx @@ -1,111 +1,88 @@ import React, { useCallback, useMemo } from 'react' -import { Button, Col, Container, Row } from 'reactstrap' -import styled from 'styled-components' +import { Button } from 'reactstrap' +import styled, { useTheme } from 'styled-components' import { ImCross } from 'react-icons/im' -import { rgba } from 'polished' - import { AlgorithmInput } from 'src/types' import { ButtonTransparent } from 'src/components/Common/ButtonTransparent' import { useTranslationSafe } from 'src/helpers/useTranslationSafe' import { useQuerySeqInputs } from 'src/state/inputs.state' - -const SequencesCurrentWrapper = styled(Container)` - border: 1px #ccc9 solid; - border-radius: 5px; -` - -const InputFileInfoWrapper = styled.section` - box-shadow: ${(props) => `1px 1px 5px ${rgba(props.theme.black, 0.1)}`}; - border: 1px #ccc9 solid; - border-radius: 5px; - margin: 0.5rem 0; - padding: 0.5rem 1rem; -` - -export interface InputFileInfoProps { - input: AlgorithmInput - index: number -} - -export function InputFileInfo({ input, index }: InputFileInfoProps) { - const { t } = useTranslationSafe() - const { removeQryInput } = useQuerySeqInputs() - const onRemoveClicked = useCallback(() => { - removeQryInput(index) - }, [index, removeQryInput]) - - return ( - - - {input.description} - - - - - - ) -} +import { UlGeneric } from '../Common/List' export function QuerySequenceList() { const { t } = useTranslationSafe() const { qryInputs, clearQryInputs } = useQuerySeqInputs() - const inputComponents = useMemo( - () => ( - - - {qryInputs.map((input, index) => ( - // eslint-disable-next-line react/no-array-index-key - - ))} - - - ), - [qryInputs], - ) - - const removeButton = useMemo( - () => - qryInputs.length > 0 ? ( - - - - - - ) : null, + const listItems = useMemo(() => { + return qryInputs.map((input, index) => ( +
  • + +
    • + )) + }, [qryInputs]) - [clearQryInputs, qryInputs.length, t], - ) const headerText = useMemo(() => { if (qryInputs.length === 0) { return null } return ( - - -

    {t("Sequence data you've added")}

    - -     +
    +

    {t("Sequence data you've added")}

    + +
    ) - }, [qryInputs.length, t]) + }, [clearQryInputs, qryInputs.length, t]) if (qryInputs.length === 0) { return null } return ( -
    + <> {headerText} - - - - {inputComponents} - {removeButton} - - - -
    +
      {listItems}
    + ) } + +export const Ul = styled(UlGeneric)` + flex: 1; + overflow: auto; +` + +export const Li = styled.li` + margin: 5px 0; + border-radius: 5px !important; +` + +export interface InputFileInfoProps { + input: AlgorithmInput + index: number +} + +export function InputFileInfo({ input, index }: InputFileInfoProps) { + const { t } = useTranslationSafe() + const theme = useTheme() + const { removeQryInput } = useQuerySeqInputs() + const onRemoveClicked = useCallback(() => { + removeQryInput(index) + }, [index, removeQryInput]) + + return ( + +
    {input.description}
    + + + +     + ) +} + +const Container = styled.section` + display: flex; + padding: 0.5rem 1rem; + box-shadow: 0 0 12px 0 #0002; + border: 1px #ccc9 solid; + border-radius: 5px; +` diff --git a/packages_rs/nextclade-web/src/components/Main/RunPanel.tsx b/packages_rs/nextclade-web/src/components/Main/RunPanel.tsx new file mode 100644 index 000000000..9e573dc0d --- /dev/null +++ b/packages_rs/nextclade-web/src/components/Main/RunPanel.tsx @@ -0,0 +1,132 @@ +import React, { useCallback, useMemo } from 'react' +import styled from 'styled-components' +import { Button, Form as FormBase, FormGroup } from 'reactstrap' +import { useRecoilValue } from 'recoil' +import { useRunAnalysis } from 'src/hooks/useRunAnalysis' +import { useRunSeqAutodetect } from 'src/hooks/useRunSeqAutodetect' +import { useRecoilToggle } from 'src/hooks/useToggle' +import { canRunAtom } from 'src/state/results.state' +import { datasetCurrentAtom } from 'src/state/dataset.state' +import { hasInputErrorsAtom } from 'src/state/error.state' +import { shouldRunAutomaticallyAtom, shouldSuggestDatasetsAtom } from 'src/state/settings.state' +import { Toggle } from 'src/components/Common/Toggle' +import { FlexLeft, FlexRight } from 'src/components/FilePicker/FilePickerStyles' +import { useTranslationSafe } from 'src/helpers/useTranslationSafe' +import { AlgorithmInputDefault } from 'src/io/AlgorithmInput' +import { hasRequiredInputsAtom, useQuerySeqInputs } from 'src/state/inputs.state' + +export function RunPanel() { + const { t } = useTranslationSafe() + + const datasetCurrent = useRecoilValue(datasetCurrentAtom) + const { addQryInputs } = useQuerySeqInputs() + + const canRun = useRecoilValue(canRunAtom) + const { state: shouldRunAutomatically, toggle: toggleRunAutomatically } = useRecoilToggle(shouldRunAutomaticallyAtom) + const shouldSuggestDatasets = useRecoilValue(shouldSuggestDatasetsAtom) + + const hasRequiredInputs = useRecoilValue(hasRequiredInputsAtom) + const hasInputErrors = useRecoilValue(hasInputErrorsAtom) + + const runAnalysis = useRunAnalysis() + const runAutodetect = useRunSeqAutodetect() + + const setExampleSequences = useCallback(() => { + if (datasetCurrent) { + addQryInputs([new AlgorithmInputDefault(datasetCurrent)]) + if (shouldSuggestDatasets) { + runAutodetect() + } + if (shouldRunAutomatically) { + runAnalysis() + } + } + }, [addQryInputs, datasetCurrent, runAnalysis, runAutodetect, shouldRunAutomatically, shouldSuggestDatasets]) + + const { isRunButtonDisabled, runButtonColor, runButtonTooltip } = useMemo(() => { + const isRunButtonDisabled = !(canRun && hasRequiredInputs) || hasInputErrors + return { + isRunButtonDisabled, + runButtonColor: isRunButtonDisabled ? 'secondary' : 'success', + runButtonTooltip: isRunButtonDisabled + ? t('Please provide sequence data for the algorithm') + : t('Launch the algorithm!'), + } + }, [canRun, hasInputErrors, hasRequiredInputs, t]) + + return ( + +
    + + + + + {t('Run automatically')} + + + + + + + + + + {t('Run')} + + +
    +     + ) +} + +const Container = styled.div` + flex: 1; + margin-top: auto; + margin-bottom: 7px; + padding: 7px 0; + padding-right: 5px; +` + +const Form = styled(FormBase)` + display: flex; + width: 100%; + height: 100%; + margin-top: auto; + padding: 10px; + border: 1px #ccc9 solid; + border-radius: 5px; +` + +// const Container = styled.div` +// flex: 1; +// margin-top: auto; +// margin-bottom: 7px; +// padding: 10px; +// padding-right: 5px; +// box-shadow: 0 3px 20px 3px #0003; +// ` +// +// const Form = styled(FormBase)` +// display: flex; +// width: 100%; +// height: 100%; +// padding: 10px; +// border: 1px #ccc9 solid; +// border-radius: 5px; +// ` + +const ButtonRunStyled = styled(Button)` + min-width: 150px; + min-height: 45px; +` diff --git a/packages_rs/nextclade-web/src/components/Main/SuggestionPanel.tsx b/packages_rs/nextclade-web/src/components/Main/SuggestionPanel.tsx new file mode 100644 index 000000000..8d9800c17 --- /dev/null +++ b/packages_rs/nextclade-web/src/components/Main/SuggestionPanel.tsx @@ -0,0 +1,101 @@ +import { isNil } from 'lodash' +import React, { useMemo } from 'react' +import { useRunSeqAutodetect } from 'src/hooks/useRunSeqAutodetect' +import { hasRequiredInputsAtom } from 'src/state/inputs.state' +import styled from 'styled-components' +import { Button, Form as FormBase, FormGroup } from 'reactstrap' +import { useRecoilValue, useResetRecoilState } from 'recoil' +import { Toggle } from 'src/components/Common/Toggle' +import { FlexLeft, FlexRight } from 'src/components/FilePicker/FilePickerStyles' +import { useTranslationSafe } from 'src/helpers/useTranslationSafe' +import { useRecoilToggle } from 'src/hooks/useToggle' +import { autodetectResultsAtom, hasAutodetectResultsAtom } from 'src/state/autodetect.state' +import { minimizerIndexVersionAtom } from 'src/state/dataset.state' +import { shouldSuggestDatasetsAtom } from 'src/state/settings.state' + +export function SuggestionPanel() { + const { t } = useTranslationSafe() + const minimizerIndexVersion = useRecoilValue(minimizerIndexVersionAtom) + const resetAutodetectResults = useResetRecoilState(autodetectResultsAtom) + const hasAutodetectResults = useRecoilValue(hasAutodetectResultsAtom) + const hasRequiredInputs = useRecoilValue(hasRequiredInputsAtom) + const runSuggest = useRunSeqAutodetect() + + const { canRun, runButtonColor, runButtonTooltip } = useMemo(() => { + const canRun = hasRequiredInputs + return { + canRun, + runButtonColor: !canRun ? 'secondary' : 'success', + runButtonTooltip: !canRun ? t('Please provide sequence data for the algorithm') : t('Launch suggestions engine!'), + } + }, [hasRequiredInputs, t]) + + if (isNil(minimizerIndexVersion)) { + return null + } + + return ( + +
    + + + + + + + + + {t('Suggest')} + + +
    +     + ) +} + +const Container = styled.div` + flex: 1; + margin-top: auto; + margin-bottom: 7px; + padding: 7px 0; + padding-left: 5px; +` + +const Form = styled(FormBase)` + display: flex; + width: 100%; + height: 100%; + margin-top: auto; + padding: 10px; + border: 1px #ccc9 solid; + border-radius: 5px; +` + +const ButtonRunStyled = styled(Button)` + min-width: 150px; + min-height: 45px; +` + +function AutosuggestionToggle() { + const { t } = useTranslationSafe() + const { state: shouldSuggestDatasets, toggle: toggleSuggestDatasets } = useRecoilToggle(shouldSuggestDatasetsAtom) + return ( + + + + {t('Suggest automatically')} + + + + ) +} diff --git a/packages_rs/nextclade-web/src/state/dataset.state.ts b/packages_rs/nextclade-web/src/state/dataset.state.ts index 913ffee91..880acf271 100644 --- a/packages_rs/nextclade-web/src/state/dataset.state.ts +++ b/packages_rs/nextclade-web/src/state/dataset.state.ts @@ -3,7 +3,6 @@ import { atom, DefaultValue, selector } from 'recoil' import type { Dataset, MinimizerIndexVersion } from 'src/types' // import { GENE_OPTION_NUC_SEQUENCE } from 'src/constants' -import { inputResetAtom } from 'src/state/inputs.state' import { persistAtom } from 'src/state/persist/localStorage' // import { viewedGeneAtom } from 'src/state/seqViewSettings.state' import { isDefaultValue } from 'src/state/utils/isDefaultValue' @@ -41,7 +40,6 @@ export const datasetCurrentAtom = selector({ set(datasetCurrentStorageAtom, dataset) // FIXME // set(viewedGeneAtom, dataset?.defaultGene ?? GENE_OPTION_NUC_SEQUENCE) - reset(inputResetAtom) } }, }) diff --git a/packages_rs/nextclade-web/src/state/inputs.state.ts b/packages_rs/nextclade-web/src/state/inputs.state.ts index be09c858f..2ddff2b13 100644 --- a/packages_rs/nextclade-web/src/state/inputs.state.ts +++ b/packages_rs/nextclade-web/src/state/inputs.state.ts @@ -1,6 +1,7 @@ import { isEmpty } from 'lodash' import { useCallback } from 'react' import { atom, selector, useRecoilState, useResetRecoilState } from 'recoil' +import { autodetectResultsAtom } from 'src/state/autodetect.state' import { AlgorithmInput } from 'src/types' import { notUndefinedOrNull } from 'src/helpers/notUndefined' @@ -11,7 +12,8 @@ export const qrySeqInputsStorageAtom = atom({ export function useQuerySeqInputs() { const [qryInputs, setQryInputs] = useRecoilState(qrySeqInputsStorageAtom) - const clearQryInputs = useResetRecoilState(qrySeqInputsStorageAtom) + const resetSeqInputsStorage = useResetRecoilState(qrySeqInputsStorageAtom) + const resetAutodetectResults = useResetRecoilState(autodetectResultsAtom) const addQryInputs = useCallback( (newInputs: AlgorithmInput[]) => { @@ -27,6 +29,11 @@ export function useQuerySeqInputs() { [setQryInputs], ) + const clearQryInputs = useCallback(() => { + resetAutodetectResults() + resetSeqInputsStorage() + }, [resetAutodetectResults, resetSeqInputsStorage]) + return { qryInputs, addQryInputs, removeQryInput, clearQryInputs } } From cabb8247aaa958e8acbf9093384c2c3e5e7ff061 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Fri, 15 Sep 2023 10:24:34 +0200 Subject: [PATCH 44/51] feat(web): prettify nav bar --- packages_rs/nextclade-web/package.json | 2 +- .../src/components/FilePicker/FilePicker.tsx | 6 +- .../components/Layout/LanguageSwitcher.tsx | 64 ++++++++++++----- .../src/components/Layout/NavigationBar.tsx | 28 +++++--- packages_rs/nextclade-web/src/i18n/i18n.ts | 72 ++++++------------- .../styles/components/LanguageSwitcher.scss | 19 ----- .../nextclade-web/src/styles/global.scss | 1 - packages_rs/nextclade-web/yarn.lock | 8 +-- 8 files changed, 97 insertions(+), 103 deletions(-) delete mode 100644 packages_rs/nextclade-web/src/styles/components/LanguageSwitcher.scss diff --git a/packages_rs/nextclade-web/package.json b/packages_rs/nextclade-web/package.json index 0c7446ac1..0f93187d9 100644 --- a/packages_rs/nextclade-web/package.json +++ b/packages_rs/nextclade-web/package.json @@ -133,7 +133,7 @@ "react-file-icon": "1.1.0", "react-helmet": "6.1.0", "react-i18next": "11.3.3", - "react-icons": "4.3.1", + "react-icons": "4.11.0", "react-if": "4.1.4", "react-loader-spinner": "5.1.4", "react-markdown": "6.0.3", diff --git a/packages_rs/nextclade-web/src/components/FilePicker/FilePicker.tsx b/packages_rs/nextclade-web/src/components/FilePicker/FilePicker.tsx index cdb00d85c..5e55a0109 100644 --- a/packages_rs/nextclade-web/src/components/FilePicker/FilePicker.tsx +++ b/packages_rs/nextclade-web/src/components/FilePicker/FilePicker.tsx @@ -39,7 +39,11 @@ export const FilePickerTitle = styled.h4` margin: auto 0; ` -export const TabsPanelStyled = styled(TabsPanel)`` +export const TabsPanelStyled = styled(TabsPanel)` + * { + background: transparent !important; + } +` const TabsContentStyled = styled(TabsContent)` height: 100%; diff --git a/packages_rs/nextclade-web/src/components/Layout/LanguageSwitcher.tsx b/packages_rs/nextclade-web/src/components/Layout/LanguageSwitcher.tsx index d0455bf72..a788f45ec 100644 --- a/packages_rs/nextclade-web/src/components/Layout/LanguageSwitcher.tsx +++ b/packages_rs/nextclade-web/src/components/Layout/LanguageSwitcher.tsx @@ -1,7 +1,13 @@ import React, { useCallback, useMemo, useState } from 'react' -import { Dropdown, DropdownToggle, DropdownMenu, DropdownItem, DropdownProps } from 'reactstrap' +import { + Dropdown as DropdownBase, + DropdownToggle as DropdownToggleBase, + DropdownMenu as DropdownMenuBase, + DropdownItem, + DropdownProps, +} from 'reactstrap' import { useRecoilState } from 'recoil' - +import styled from 'styled-components' import { localeAtom } from 'src/state/locale.state' import { getLocaleWithKey, Locale, localesArray } from 'src/i18n/i18n' @@ -14,11 +20,11 @@ export function LanguageSwitcher({ ...restProps }: LanguageSwitcherProps) { const setLocaleLocal = useCallback((locale: Locale) => () => setCurrentLocale(locale.key), [setCurrentLocale]) return ( - + - + - + {localesArray.map((locale) => { const isCurrent = locale.key === currentLocale return ( @@ -33,20 +39,42 @@ export function LanguageSwitcher({ ...restProps }: LanguageSwitcherProps) { } export function LanguageSwitcherItem({ locale }: { locale: string }) { - const { Flag, name, native } = getLocaleWithKey(locale) - - const label = useMemo(() => { - if (name === native) { - return name - } - - return `${native} (${name})` + const { name, native } = getLocaleWithKey(locale) + const { label, tooltip } = useMemo(() => { + return { label: `(${native})`, tooltip: `${name} (${native})` } }, [name, native]) - return ( - <> - - {label} - + + + {label} + ) } + +export function LabelShort({ locale, ...restProps }: { locale: string; className?: string }) { + const { key } = getLocaleWithKey(locale) + return {key} +} + +const LabelShortText = styled.span` + font-family: ${(props) => props.theme.font.monospace}; + text-transform: uppercase !important; + color: unset !important; +` + +const Dropdown = styled(DropdownBase)` + padding: 0; + margin: 0; +` + +const DropdownToggle = styled(DropdownToggleBase)` + color: ${(props) => props.theme.bodyColor}; + padding: 0; + margin: 0; +` + +const DropdownMenu = styled(DropdownMenuBase)` + background-color: ${(props) => props.theme.bodyBg}; + box-shadow: 1px 1px 20px 0 #0005; + transition: opacity ease-out 0.25s; +` diff --git a/packages_rs/nextclade-web/src/components/Layout/NavigationBar.tsx b/packages_rs/nextclade-web/src/components/Layout/NavigationBar.tsx index b7933b6b3..89f737e01 100644 --- a/packages_rs/nextclade-web/src/components/Layout/NavigationBar.tsx +++ b/packages_rs/nextclade-web/src/components/Layout/NavigationBar.tsx @@ -8,7 +8,7 @@ import { } from 'reactstrap' import { useRecoilValue } from 'recoil' import { Link } from 'src/components/Link/Link' -import { FaDocker, FaGithub, FaTwitter } from 'react-icons/fa' +import { FaDocker, FaGithub, FaXTwitter, FaDiscourse } from 'react-icons/fa6' import { LinkSmart } from 'src/components/Link/LinkSmart' import { hasRanAtom, hasTreeAtom } from 'src/state/results.state' import styled from 'styled-components' @@ -18,15 +18,14 @@ import { CitationButton } from 'src/components/Citation/CitationButton' import { NextcladeTextLogo } from 'src/components/Layout/NextcladeTextLogo' import { LanguageSwitcher } from './LanguageSwitcher' -const LOGO_SIZE = 30 +const LOGO_SIZE = 36 export const Navbar = styled(NavbarBase)` - height: 38px; + height: 45px; display: flex; - vertical-align: middle; padding: 0 !important; margin: 0 !important; - box-shadow: ${(props) => props.theme.shadows.large}; + box-shadow: 0 0 8px 0 #0004; ` export const Nav = styled(NavBase)` @@ -40,7 +39,11 @@ export const NavItem = styled(NavItemBase)` padding: 0 0.5rem; flex-grow: 0; flex-shrink: 0; - margin: 0 !important; + margin: auto; + + * { + vertical-align: middle; + } ` const NavbarBrand = styled(NavbarBrandBase)` @@ -138,18 +141,23 @@ export function NavigationBar() { }, { url: 'https://twitter.com/nextstrain', - title: t('Link to our Twitter'), - content: , + title: t('Link to our X.com (Twitter)'), + content: , + }, + { + url: 'https://discussion.nextstrain.org/', + title: t('Link to our discussion forum'), + content: , }, { url: 'https://hub.docker.com/r/nextstrain/nextclade', title: t('Link to our Docker containers'), - content: , + content: , }, { url: 'https://github.com/nextstrain/nextclade', title: t('Link to our Github page'), - content: , + content: , }, { title: t('Change language'), diff --git a/packages_rs/nextclade-web/src/i18n/i18n.ts b/packages_rs/nextclade-web/src/i18n/i18n.ts index 7dfa189e8..abb25b55c 100644 --- a/packages_rs/nextclade-web/src/i18n/i18n.ts +++ b/packages_rs/nextclade-web/src/i18n/i18n.ts @@ -1,11 +1,7 @@ -import { ElementType, FC } from 'react' - import type { StrictOmit } from 'ts-essentials' import { get, isNil, mapValues } from 'lodash' - import i18nOriginal, { i18n as I18N, Resource } from 'i18next' import { initReactI18next } from 'react-i18next' - import { Settings as LuxonSettings } from 'luxon' import numbro from 'numbro' import { languages } from 'countries-list' @@ -15,28 +11,6 @@ import prettyBytesOriginal, { Options as PrettyBytesOptionsOriginal } from 'pret // @ts-ignore import numbroLanguages from 'numbro/dist/languages.min' -import CN from 'flag-icon-css/flags/1x1/cn.svg' -import DE from 'flag-icon-css/flags/1x1/de.svg' -import ES from 'flag-icon-css/flags/1x1/es.svg' -import FR from 'flag-icon-css/flags/1x1/fr.svg' -import GB from 'flag-icon-css/flags/1x1/gb.svg' -import GR from 'flag-icon-css/flags/1x1/gr.svg' -import ID from 'flag-icon-css/flags/1x1/id.svg' -import IL from 'flag-icon-css/flags/1x1/il.svg' -import IN from 'flag-icon-css/flags/1x1/in.svg' -import IR from 'flag-icon-css/flags/1x1/ir.svg' -import IT from 'flag-icon-css/flags/1x1/it.svg' -import JP from 'flag-icon-css/flags/1x1/jp.svg' -import KR from 'flag-icon-css/flags/1x1/kr.svg' -import NL from 'flag-icon-css/flags/1x1/nl.svg' -import PK from 'flag-icon-css/flags/1x1/pk.svg' -import PT from 'flag-icon-css/flags/1x1/pt.svg' -import RU from 'flag-icon-css/flags/1x1/ru.svg' -import SA from 'flag-icon-css/flags/1x1/sa.svg' -import TH from 'flag-icon-css/flags/1x1/th.svg' -import TR from 'flag-icon-css/flags/1x1/tr.svg' -import VN from 'flag-icon-css/flags/1x1/vn.svg' - import ar from './resources/ar/common.json' import de from './resources/de/common.json' import el from './resources/el/common.json' @@ -97,32 +71,32 @@ export interface Locale { readonly full: string readonly name: string readonly native: string - readonly Flag: ElementType + readonly rtl: number | undefined } export const locales: Record = { - en: { key: 'en', full: 'en-US', name: languages.en.name, native: languages.en.native, Flag: GB as FC }, - ar: { key: 'ar', full: 'ar-SA', name: languages.ar.name, native: languages.ar.native, Flag: SA as FC }, - de: { key: 'de', full: 'de-DE', name: languages.de.name, native: languages.de.native, Flag: DE as FC }, - el: { key: 'el', full: 'el-GR', name: languages.el.name, native: languages.el.native, Flag: GR as FC }, - es: { key: 'es', full: 'es-ES', name: languages.es.name, native: languages.es.native, Flag: ES as FC }, - fa: { key: 'fa', full: 'fa-IR', name: languages.fa.name, native: languages.fa.native, Flag: IR as FC }, - fr: { key: 'fr', full: 'fr-FR', name: languages.fr.name, native: languages.fr.native, Flag: FR as FC }, - he: { key: 'he', full: 'he-IL', name: languages.he.name, native: languages.he.native, Flag: IL as FC }, - hi: { key: 'hi', full: 'hi-IN', name: languages.hi.name, native: languages.hi.native, Flag: IN as FC }, - id: { key: 'id', full: 'id-ID', name: languages.id.name, native: languages.id.native, Flag: ID as FC }, - it: { key: 'it', full: 'it-IT', name: languages.it.name, native: languages.it.native, Flag: IT as FC }, - ja: { key: 'ja', full: 'ja-JP', name: languages.ja.name, native: languages.ja.native, Flag: JP as FC }, - ko: { key: 'ko', full: 'ko-KR', name: languages.ko.name, native: languages.ko.native, Flag: KR as FC }, - nl: { key: 'nl', full: 'nl-NL', name: languages.nl.name, native: languages.nl.native, Flag: NL as FC }, - pt: { key: 'pt', full: 'pt-PT', name: languages.pt.name, native: languages.pt.native, Flag: PT as FC }, - ru: { key: 'ru', full: 'ru-RU', name: languages.ru.name, native: languages.ru.native, Flag: RU as FC }, - ta: { key: 'ta', full: 'ta-IN', name: languages.ta.name, native: languages.ta.native, Flag: IN as FC }, - th: { key: 'th', full: 'th-TH', name: languages.th.name, native: languages.th.native, Flag: TH as FC }, - tr: { key: 'tr', full: 'tr-TR', name: languages.tr.name, native: languages.tr.native, Flag: TR as FC }, - ur: { key: 'ur', full: 'ur-PK', name: languages.ur.name, native: languages.ur.native, Flag: PK as FC }, - vi: { key: 'vi', full: 'vi-VN', name: languages.vi.name, native: languages.vi.native, Flag: VN as FC }, - zh: { key: 'zh', full: 'zh-CN', name: languages.zh.name, native: languages.zh.native, Flag: CN as FC }, + en: { key: 'en', full: 'en-US', name: languages.en.name, native: languages.en.native, rtl: languages.en.rtl }, + ar: { key: 'ar', full: 'ar-SA', name: languages.ar.name, native: languages.ar.native, rtl: languages.ar.rtl }, + de: { key: 'de', full: 'de-DE', name: languages.de.name, native: languages.de.native, rtl: languages.de.rtl }, + el: { key: 'el', full: 'el-GR', name: languages.el.name, native: languages.el.native, rtl: languages.el.rtl }, + es: { key: 'es', full: 'es-ES', name: languages.es.name, native: languages.es.native, rtl: languages.es.rtl }, + fa: { key: 'fa', full: 'fa-IR', name: languages.fa.name, native: languages.fa.native, rtl: languages.fa.rtl }, + fr: { key: 'fr', full: 'fr-FR', name: languages.fr.name, native: languages.fr.native, rtl: languages.fr.rtl }, + he: { key: 'he', full: 'he-IL', name: languages.he.name, native: languages.he.native, rtl: languages.he.rtl }, + hi: { key: 'hi', full: 'hi-IN', name: languages.hi.name, native: languages.hi.native, rtl: languages.hi.rtl }, + id: { key: 'id', full: 'id-ID', name: languages.id.name, native: languages.id.native, rtl: languages.id.rtl }, + it: { key: 'it', full: 'it-IT', name: languages.it.name, native: languages.it.native, rtl: languages.it.rtl }, + ja: { key: 'ja', full: 'ja-JP', name: languages.ja.name, native: languages.ja.native, rtl: languages.ja.rtl }, + ko: { key: 'ko', full: 'ko-KR', name: languages.ko.name, native: languages.ko.native, rtl: languages.ko.rtl }, + nl: { key: 'nl', full: 'nl-NL', name: languages.nl.name, native: languages.nl.native, rtl: languages.nl.rtl }, + pt: { key: 'pt', full: 'pt-PT', name: languages.pt.name, native: languages.pt.native, rtl: languages.pt.rtl }, + ru: { key: 'ru', full: 'ru-RU', name: languages.ru.name, native: languages.ru.native, rtl: languages.ru.rtl }, + ta: { key: 'ta', full: 'ta-IN', name: languages.ta.name, native: languages.ta.native, rtl: languages.ta.rtl }, + th: { key: 'th', full: 'th-TH', name: languages.th.name, native: languages.th.native, rtl: languages.th.rtl }, + tr: { key: 'tr', full: 'tr-TR', name: languages.tr.name, native: languages.tr.native, rtl: languages.tr.rtl }, + ur: { key: 'ur', full: 'ur-PK', name: languages.ur.name, native: languages.ur.native, rtl: languages.ur.rtl }, + vi: { key: 'vi', full: 'vi-VN', name: languages.vi.name, native: languages.vi.native, rtl: languages.vi.rtl }, + zh: { key: 'zh', full: 'zh-CN', name: languages.zh.name, native: languages.zh.native, rtl: languages.zh.rtl }, } as const export const localeKeys = Object.keys(locales) diff --git a/packages_rs/nextclade-web/src/styles/components/LanguageSwitcher.scss b/packages_rs/nextclade-web/src/styles/components/LanguageSwitcher.scss deleted file mode 100644 index bd2f66132..000000000 --- a/packages_rs/nextclade-web/src/styles/components/LanguageSwitcher.scss +++ /dev/null @@ -1,19 +0,0 @@ -@import '../variables'; - -.language-switcher { - margin-top: 3px; - - .language-switcher-menu.dropdown-menu { - background-color: $body-bg; - box-shadow: 1px 1px 2px 2px rgba($gray-600, 0.25); - } - - .language-switcher-flag { - width: 20px; - height: 20px; - margin-bottom: 5px; - background-size: cover; - border-radius: 2px; - box-shadow: 1px 1px 2px 2px rgba($gray-600, 0.25); - } -} diff --git a/packages_rs/nextclade-web/src/styles/global.scss b/packages_rs/nextclade-web/src/styles/global.scss index 5be3cef65..a8f6a4289 100644 --- a/packages_rs/nextclade-web/src/styles/global.scss +++ b/packages_rs/nextclade-web/src/styles/global.scss @@ -9,7 +9,6 @@ @import './auspice'; -@import './components/LanguageSwitcher'; @import './components/Results'; html, body, #__next { diff --git a/packages_rs/nextclade-web/yarn.lock b/packages_rs/nextclade-web/yarn.lock index 5c15055fe..2499a6d2d 100644 --- a/packages_rs/nextclade-web/yarn.lock +++ b/packages_rs/nextclade-web/yarn.lock @@ -13274,10 +13274,10 @@ react-i18next@11.3.3, react-i18next@^11.3.3: "@babel/runtime" "^7.3.1" html-parse-stringify2 "2.0.1" -react-icons@4.3.1: - version "4.3.1" - resolved "https://registry.yarnpkg.com/react-icons/-/react-icons-4.3.1.tgz#2fa92aebbbc71f43d2db2ed1aed07361124e91ca" - integrity sha512-cB10MXLTs3gVuXimblAdI71jrJx8njrJZmNMEMC+sQu5B/BIOmlsAjskdqpn81y8UBVEGuHODd7/ci5DvoSzTQ== +react-icons@4.11.0: + version "4.11.0" + resolved "https://registry.yarnpkg.com/react-icons/-/react-icons-4.11.0.tgz#4b0e31c9bfc919608095cc429c4f1846f4d66c65" + integrity sha512-V+4khzYcE5EBk/BvcuYRq6V/osf11ODUM2J8hg2FDSswRrGvqiYUYPRy4OdrWaQOBj4NcpJfmHZLNaD+VH0TyA== react-icons@^3.9.0: version "3.11.0" From 55093f961b02b8ba478c2cb02b40ea29d13103e6 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Fri, 15 Sep 2023 10:28:35 +0200 Subject: [PATCH 45/51] fix(web): small styling --- .../nextclade-web/src/components/FilePicker/UploadBox.tsx | 4 ++-- .../src/components/Main/QuerySequenceFilePicker.tsx | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages_rs/nextclade-web/src/components/FilePicker/UploadBox.tsx b/packages_rs/nextclade-web/src/components/FilePicker/UploadBox.tsx index d559604e2..42a711b95 100644 --- a/packages_rs/nextclade-web/src/components/FilePicker/UploadBox.tsx +++ b/packages_rs/nextclade-web/src/components/FilePicker/UploadBox.tsx @@ -56,6 +56,7 @@ export const UploadZoneTextContainer = styled.div` export const UploadZoneText = styled.div` font-size: 1.1rem; text-align: center; + max-width: 150px; ` export const UploadZoneButton = styled(Button)` @@ -83,8 +84,7 @@ export function UploadBox({ onUpload, children, multiple = false, ...props }: Pr const normal = useMemo( () => ( - {t('Drag & drop files')} - {t('or folders')} + {t('Drag & drop files or folders')} {t('Select files')} ), diff --git a/packages_rs/nextclade-web/src/components/Main/QuerySequenceFilePicker.tsx b/packages_rs/nextclade-web/src/components/Main/QuerySequenceFilePicker.tsx index c0f3e4bc5..0c69f6f48 100644 --- a/packages_rs/nextclade-web/src/components/Main/QuerySequenceFilePicker.tsx +++ b/packages_rs/nextclade-web/src/components/Main/QuerySequenceFilePicker.tsx @@ -82,6 +82,7 @@ const Container = styled.div` height: 100%; overflow: hidden; margin-left: 10px; + margin-right: 12px; ` const Header = styled.div` From 68438ff7464afd1627c21a587f775fad405d7b5a Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Fri, 15 Sep 2023 12:15:41 +0200 Subject: [PATCH 46/51] fix(web): use unique files ids as react keys --- packages_rs/nextclade-web/package.json | 3 +- .../src/components/FilePicker/FilePicker.tsx | 6 ++-- .../src/components/Main/QuerySequenceList.tsx | 2 +- .../nextclade-web/src/helpers/uniqueId.ts | 5 ++++ .../nextclade-web/src/io/AlgorithmInput.ts | 29 ++++++++++++++----- packages_rs/nextclade-web/src/types.ts | 2 ++ packages_rs/nextclade-web/yarn.lock | 5 ++++ 7 files changed, 39 insertions(+), 13 deletions(-) create mode 100644 packages_rs/nextclade-web/src/helpers/uniqueId.ts diff --git a/packages_rs/nextclade-web/package.json b/packages_rs/nextclade-web/package.json index 0f93187d9..7c5b95673 100644 --- a/packages_rs/nextclade-web/package.json +++ b/packages_rs/nextclade-web/package.json @@ -117,6 +117,7 @@ "luxon": "2.3.2", "marked": "4.0.14", "memoize-one": "6.0.0", + "nanoid": "3.3.6", "next": "12.1.6", "next-compose-plugins": "2.2.1", "numbro": "2.3.6", @@ -239,13 +240,13 @@ "allow-methods": "3.1.0", "babel-plugin-parameter-decorator": "1.0.16", "babel-plugin-transform-typescript-metadata": "0.3.2", + "commander": "10.0.1", "compression-webpack-plugin": "9.2.0", "connect-history-api-fallback": "1.6.0", "conventional-changelog-cli": "2.2.2", "copy-webpack-plugin": "10.2.4", "cross-env": "7.0.3", "css-loader": "6.7.1", - "commander": "10.0.1", "dotenv": "16.0.0", "eslint": "8.14.0", "eslint-config-airbnb": "19.0.4", diff --git a/packages_rs/nextclade-web/src/components/FilePicker/FilePicker.tsx b/packages_rs/nextclade-web/src/components/FilePicker/FilePicker.tsx index 5e55a0109..87aaf3d68 100644 --- a/packages_rs/nextclade-web/src/components/FilePicker/FilePicker.tsx +++ b/packages_rs/nextclade-web/src/components/FilePicker/FilePicker.tsx @@ -111,12 +111,12 @@ export function FilePicker({ const onPaste = useCallback( (content: string) => { if (multiple) { - onInputs?.([new AlgorithmInputString(content)]) + onInputs?.([new AlgorithmInputString(content, t('Pasted sequences'))]) } else { - onInput?.(new AlgorithmInputString(content)) + onInput?.(new AlgorithmInputString(content, t('Pasted sequences'))) } }, - [multiple, onInput, onInputs], + [multiple, onInput, onInputs, t], ) // eslint-disable-next-line no-void diff --git a/packages_rs/nextclade-web/src/components/Main/QuerySequenceList.tsx b/packages_rs/nextclade-web/src/components/Main/QuerySequenceList.tsx index cb7802f94..61b0f91eb 100644 --- a/packages_rs/nextclade-web/src/components/Main/QuerySequenceList.tsx +++ b/packages_rs/nextclade-web/src/components/Main/QuerySequenceList.tsx @@ -14,7 +14,7 @@ export function QuerySequenceList() { const listItems = useMemo(() => { return qryInputs.map((input, index) => ( -
  • +
    • )) diff --git a/packages_rs/nextclade-web/src/helpers/uniqueId.ts b/packages_rs/nextclade-web/src/helpers/uniqueId.ts new file mode 100644 index 000000000..e1525d609 --- /dev/null +++ b/packages_rs/nextclade-web/src/helpers/uniqueId.ts @@ -0,0 +1,5 @@ +import { nanoid } from 'nanoid' + +export function uniqueId(): string { + return nanoid() +} diff --git a/packages_rs/nextclade-web/src/io/AlgorithmInput.ts b/packages_rs/nextclade-web/src/io/AlgorithmInput.ts index 248d9c304..5e3d398ad 100644 --- a/packages_rs/nextclade-web/src/io/AlgorithmInput.ts +++ b/packages_rs/nextclade-web/src/io/AlgorithmInput.ts @@ -1,6 +1,6 @@ +import { uniqueId } from 'src/helpers/uniqueId' import { AlgorithmInput, AlgorithmInputType, Dataset } from 'src/types' import { axiosFetchRaw } from 'src/io/axiosFetch' - import { readFile } from 'src/helpers/readFile' import { numbro } from 'src/i18n/i18n' @@ -16,12 +16,20 @@ function formatBytes(bytes: number) { } export class AlgorithmInputFile implements AlgorithmInput { + public readonly uid = uniqueId() + public readonly path: string public readonly type: AlgorithmInputType = AlgorithmInputType.File as const - private readonly file: File constructor(file: File) { this.file = file + + // eslint-disable-next-line unicorn/prefer-ternary + if (this.file.webkitRelativePath.trim().length > 0) { + this.path = this.file.webkitRelativePath ?? `${this.uid}-${this.file}` + } else { + this.path = `${this.uid}-${this.file.name}` + } } public get name(): string { @@ -38,12 +46,14 @@ export class AlgorithmInputFile implements AlgorithmInput { } export class AlgorithmInputUrl implements AlgorithmInput { + public readonly uid = uniqueId() + public readonly path: string public readonly type: AlgorithmInputType = AlgorithmInputType.Url as const - private readonly url: string constructor(url: string) { this.url = url + this.path = this.url } public get name(): string { @@ -60,12 +70,14 @@ export class AlgorithmInputUrl implements AlgorithmInput { } export class AlgorithmInputString implements AlgorithmInput { + public readonly uid = uniqueId() + public readonly path: string public readonly type: AlgorithmInputType = AlgorithmInputType.String as const - private readonly content: string private readonly contentName: string constructor(content: string, contentName?: string) { + this.path = `pasted-${this.uid}.fasta` this.content = content this.contentName = contentName ?? 'Pasted sequences' } @@ -84,21 +96,22 @@ export class AlgorithmInputString implements AlgorithmInput { } export class AlgorithmInputDefault implements AlgorithmInput { + public readonly uid = uniqueId() + public readonly path: string public readonly type: AlgorithmInputType = AlgorithmInputType.Default as const - public dataset: Dataset constructor(dataset: Dataset) { this.dataset = dataset + this.path = `${this.dataset.path}/${this.dataset.files.examples ?? 'unknown.fasta'}` } public get name(): string { - const { value, valueFriendly } = this.dataset.attributes.name - return `${valueFriendly ?? value} example sequences` + return this.path } public get description(): string { - return `${this.name}` + return this.name } public async getContent(): Promise { diff --git a/packages_rs/nextclade-web/src/types.ts b/packages_rs/nextclade-web/src/types.ts index 6cabb3df4..03fc0a055 100644 --- a/packages_rs/nextclade-web/src/types.ts +++ b/packages_rs/nextclade-web/src/types.ts @@ -127,6 +127,8 @@ export enum AlgorithmInputType { } export interface AlgorithmInput { + uid: string + path: string type: AlgorithmInputType name: string description: string diff --git a/packages_rs/nextclade-web/yarn.lock b/packages_rs/nextclade-web/yarn.lock index 2499a6d2d..df7774898 100644 --- a/packages_rs/nextclade-web/yarn.lock +++ b/packages_rs/nextclade-web/yarn.lock @@ -11515,6 +11515,11 @@ nano-time@1.0.0: dependencies: big-integer "^1.6.16" +nanoid@3.3.6: + version "3.3.6" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.6.tgz#443380c856d6e9f9824267d960b4236ad583ea4c" + integrity sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA== + nanoid@^3.1.30, nanoid@^3.3.1: version "3.3.3" resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.3.tgz#fd8e8b7aa761fe807dba2d1b98fb7241bb724a25" From 01f3449bea950c8683158c1003ebebb52343d979 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Mon, 18 Sep 2023 08:13:54 +0200 Subject: [PATCH 47/51] feat(web): scroll current dataset list item into view --- .../components/Main/DatasetSelectorList.tsx | 70 +++++++++++++++---- 1 file changed, 57 insertions(+), 13 deletions(-) diff --git a/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx b/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx index d02a23bde..f2043b3c1 100644 --- a/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx +++ b/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx @@ -1,6 +1,6 @@ import { get, isNil, sortBy } from 'lodash' import { lighten } from 'polished' -import React, { useCallback, useMemo } from 'react' +import React, { forwardRef, useCallback, useMemo, useRef } from 'react' import { ListGroup, ListGroupItem } from 'reactstrap' import { useRecoilValue } from 'recoil' import { ListGenericCss } from 'src/components/Common/List' @@ -65,6 +65,30 @@ export function DatasetSelectorList({ const { itemsStartWith, itemsInclude, itemsNotInclude } = searchResult + const itemsRef = useRef | null>(null) + + function scrollToId(itemId: string) { + const map = getMap() + const node = map.get(itemId) + node?.scrollIntoView({ + behavior: 'smooth', + block: 'nearest', + inline: 'center', + }) + } + + function getMap() { + if (!itemsRef.current) { + // Initialize the Map on first usage. + itemsRef.current = new Map() + } + return itemsRef.current + } + + if (datasetHighlighted) { + scrollToId(datasetHighlighted.path) + } + const listItems = useMemo(() => { return ( <> @@ -72,6 +96,14 @@ export function DatasetSelectorList({ datasets.map((dataset) => ( { + const map = getMap() + if (node) { + map.set(dataset.path, node) + } else { + map.delete(dataset.path) + } + }} dataset={dataset} onClick={onItemClick(dataset)} isCurrent={areDatasetsEqual(dataset, datasetHighlighted)} @@ -83,6 +115,14 @@ export function DatasetSelectorList({ datasets.map((dataset) => ( { + const map = getMap() + if (node) { + map.set(dataset.path, node) + } else { + map.delete(dataset.path) + } + }} dataset={dataset} onClick={onItemClick(dataset)} isCurrent={areDatasetsEqual(dataset, datasetHighlighted)} @@ -105,7 +145,7 @@ export const Ul = styled(ListGroup)` border-radius: 0 !important; ` -export const Li = styled(ListGroupItem)<{ $isDimmed?: boolean }>` +export const Li = styled.li<{ $active?: boolean; $isDimmed?: boolean }>` cursor: pointer; opacity: ${(props) => props.$isDimmed && 0.4}; background-color: transparent; @@ -114,11 +154,13 @@ export const Li = styled(ListGroupItem)<{ $isDimmed?: boolean }>` padding: 0 !important; border-radius: 5px !important; - &.active { - background-color: ${(props) => lighten(0.033)(props.theme.primary)}; + ${(props) => + props.$active && + ` + background-color: ${lighten(0.033)(props.theme.primary)}; box-shadow: -3px 3px 12px 3px #0005; - opacity: ${(props) => props.$isDimmed && 0.66}; - } + opacity: ${props.$isDimmed && 0.66}; + `}; ` interface DatasetSelectorListItemProps { @@ -128,10 +170,12 @@ interface DatasetSelectorListItemProps { onClick?: () => void } -function DatasetSelectorListItem({ dataset, isCurrent, isDimmed, onClick }: DatasetSelectorListItemProps) { - return ( -
  • - -
    • - ) -} +const DatasetSelectorListItem = forwardRef( + function DatasetSelectorListItemWithRef({ dataset, isCurrent, isDimmed, onClick }, ref) { + return ( +
  • + +
    • + ) + }, +) From 453cf3b7bec41c0f26beee95c53d0e91d2e9c0b7 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Mon, 18 Sep 2023 08:24:01 +0200 Subject: [PATCH 48/51] refactor: simplify list scrolling setup --- .../components/Main/DatasetSelectorList.tsx | 43 +++++++------------ 1 file changed, 15 insertions(+), 28 deletions(-) diff --git a/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx b/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx index f2043b3c1..5cbc54481 100644 --- a/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx +++ b/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx @@ -1,7 +1,7 @@ import { get, isNil, sortBy } from 'lodash' import { lighten } from 'polished' import React, { forwardRef, useCallback, useMemo, useRef } from 'react' -import { ListGroup, ListGroupItem } from 'reactstrap' +import { ListGroup } from 'reactstrap' import { useRecoilValue } from 'recoil' import { ListGenericCss } from 'src/components/Common/List' import styled from 'styled-components' @@ -65,11 +65,10 @@ export function DatasetSelectorList({ const { itemsStartWith, itemsInclude, itemsNotInclude } = searchResult - const itemsRef = useRef | null>(null) + const itemsRef = useRef>(new Map()) function scrollToId(itemId: string) { - const map = getMap() - const node = map.get(itemId) + const node = itemsRef.current.get(itemId) node?.scrollIntoView({ behavior: 'smooth', block: 'nearest', @@ -77,14 +76,6 @@ export function DatasetSelectorList({ }) } - function getMap() { - if (!itemsRef.current) { - // Initialize the Map on first usage. - itemsRef.current = new Map() - } - return itemsRef.current - } - if (datasetHighlighted) { scrollToId(datasetHighlighted.path) } @@ -96,14 +87,7 @@ export function DatasetSelectorList({ datasets.map((dataset) => ( { - const map = getMap() - if (node) { - map.set(dataset.path, node) - } else { - map.delete(dataset.path) - } - }} + ref={nodeRefSetOrDelete(itemsRef.current, dataset.path)} dataset={dataset} onClick={onItemClick(dataset)} isCurrent={areDatasetsEqual(dataset, datasetHighlighted)} @@ -115,14 +99,7 @@ export function DatasetSelectorList({ datasets.map((dataset) => ( { - const map = getMap() - if (node) { - map.set(dataset.path, node) - } else { - map.delete(dataset.path) - } - }} + ref={nodeRefSetOrDelete(itemsRef.current, dataset.path)} dataset={dataset} onClick={onItemClick(dataset)} isCurrent={areDatasetsEqual(dataset, datasetHighlighted)} @@ -137,6 +114,16 @@ export function DatasetSelectorList({ return
      {listItems}
    } +function nodeRefSetOrDelete(map: Map, key: string) { + return function nodeRefSetOrDeleteImpl(node: T) { + if (node) { + map.set(key, node) + } else { + map.delete(key) + } + } +} + export const Ul = styled(ListGroup)` ${ListGenericCss}; flex: 1; From f0e0cdf6e390c4529eaa99457bb261bb37ec295c Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Mon, 18 Sep 2023 09:03:18 +0200 Subject: [PATCH 49/51] feat(web): preselect top dataset after suggestion is complete --- .../components/Main/DatasetSelectorList.tsx | 26 ++++++--- .../src/hooks/useRunSeqAutodetect.ts | 54 +++++++++++-------- .../src/state/autodetect.state.ts | 14 ++++- 3 files changed, 66 insertions(+), 28 deletions(-) diff --git a/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx b/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx index 5cbc54481..2bb5be734 100644 --- a/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx +++ b/packages_rs/nextclade-web/src/components/Main/DatasetSelectorList.tsx @@ -1,15 +1,20 @@ import { get, isNil, sortBy } from 'lodash' import { lighten } from 'polished' -import React, { forwardRef, useCallback, useMemo, useRef } from 'react' +import React, { forwardRef, useCallback, useEffect, useMemo, useRef } from 'react' import { ListGroup } from 'reactstrap' -import { useRecoilValue } from 'recoil' +import { useRecoilState, useRecoilValue } from 'recoil' import { ListGenericCss } from 'src/components/Common/List' -import styled from 'styled-components' +import { DatasetInfo } from 'src/components/Main/DatasetInfo' +import { search } from 'src/helpers/search' +import { + autodetectResultsAtom, + AutodetectRunState, + autodetectRunStateAtom, + groupByDatasets, +} from 'src/state/autodetect.state' import type { Dataset } from 'src/types' import { areDatasetsEqual } from 'src/types' -import { autodetectResultsAtom, groupByDatasets } from 'src/state/autodetect.state' -import { search } from 'src/helpers/search' -import { DatasetInfo } from 'src/components/Main/DatasetInfo' +import styled from 'styled-components' export interface DatasetSelectorListProps { datasets: Dataset[] @@ -28,6 +33,7 @@ export function DatasetSelectorList({ const onItemClick = useCallback((dataset: Dataset) => () => onDatasetHighlighted(dataset), [onDatasetHighlighted]) const autodetectResults = useRecoilValue(autodetectResultsAtom) + const [autodetectRunState, setAutodetectRunState] = useRecoilState(autodetectRunStateAtom) const autodetectResult = useMemo(() => { if (isNil(autodetectResults) || autodetectResults.length === 0) { @@ -80,6 +86,14 @@ export function DatasetSelectorList({ scrollToId(datasetHighlighted.path) } + useEffect(() => { + const topSuggestion = autodetectResult.itemsInclude[0] + if (autodetectRunState === AutodetectRunState.Done) { + onDatasetHighlighted(topSuggestion) + setAutodetectRunState(AutodetectRunState.Idle) + } + }, [autodetectRunState, autodetectResult.itemsInclude, onDatasetHighlighted, setAutodetectRunState]) + const listItems = useMemo(() => { return ( <> diff --git a/packages_rs/nextclade-web/src/hooks/useRunSeqAutodetect.ts b/packages_rs/nextclade-web/src/hooks/useRunSeqAutodetect.ts index ce0af4768..fb1ce5837 100644 --- a/packages_rs/nextclade-web/src/hooks/useRunSeqAutodetect.ts +++ b/packages_rs/nextclade-web/src/hooks/useRunSeqAutodetect.ts @@ -2,8 +2,15 @@ import type { Subscription } from 'observable-fns' import { useRecoilCallback } from 'recoil' import { ErrorInternal } from 'src/helpers/ErrorInternal' import { axiosFetch } from 'src/io/axiosFetch' -import { autodetectResultByIndexAtom, autodetectResultsAtom, minimizerIndexAtom } from 'src/state/autodetect.state' +import { + autodetectResultByIndexAtom, + autodetectResultsAtom, + AutodetectRunState, + autodetectRunStateAtom, + minimizerIndexAtom, +} from 'src/state/autodetect.state' import { minimizerIndexVersionAtom } from 'src/state/dataset.state' +import { globalErrorAtom } from 'src/state/error.state' import { MinimizerIndexJson, MinimizerSearchRecord } from 'src/types' import { qrySeqInputsStorageAtom } from 'src/state/inputs.state' import { getQueryFasta } from 'src/workers/launchAnalysis' @@ -12,10 +19,15 @@ import { spawn } from 'src/workers/spawn' export function useRunSeqAutodetect() { return useRecoilCallback( - ({ set, reset, snapshot: { getPromise } }) => + ({ set, reset, snapshot }) => () => { + const { getPromise } = snapshot + + set(autodetectRunStateAtom, AutodetectRunState.Started) + reset(minimizerIndexAtom) reset(autodetectResultsAtom) + reset(autodetectRunStateAtom) function onResult(results: MinimizerSearchRecord[]) { results.forEach((res) => { @@ -23,6 +35,15 @@ export function useRunSeqAutodetect() { }) } + function onError(error: Error) { + set(autodetectRunStateAtom, AutodetectRunState.Failed) + set(globalErrorAtom, error) + } + + function onComplete() { + set(autodetectRunStateAtom, AutodetectRunState.Done) + } + Promise.all([getPromise(qrySeqInputsStorageAtom), getPromise(minimizerIndexVersionAtom)]) .then(async ([qrySeqInputs, minimizerIndexVersion]) => { if (!minimizerIndexVersion) { @@ -31,7 +52,7 @@ export function useRunSeqAutodetect() { const fasta = await getQueryFasta(qrySeqInputs) const minimizerIndex: MinimizerIndexJson = await axiosFetch(minimizerIndexVersion.path) set(minimizerIndexAtom, minimizerIndex) - return runAutodetect(fasta, minimizerIndex, onResult) + return runAutodetect(fasta, minimizerIndex, { onResult, onError, onComplete }) }) .catch((error) => { throw error @@ -41,13 +62,15 @@ export function useRunSeqAutodetect() { ) } -async function runAutodetect( - fasta: string, - minimizerIndex: MinimizerIndexJson, - onResult: (res: MinimizerSearchRecord[]) => void, -) { +interface Callbacks { + onResult: (r: MinimizerSearchRecord[]) => void + onError?: (error: Error) => void + onComplete?: () => void +} + +async function runAutodetect(fasta: string, minimizerIndex: MinimizerIndexJson, callbacks: Callbacks) { const worker = await SeqAutodetectWasmWorker.create(minimizerIndex) - await worker.autodetect(fasta, { onResult }) + await worker.autodetect(fasta, callbacks) await worker.destroy() } @@ -73,18 +96,7 @@ export class SeqAutodetectWasmWorker { await this.thread.create(minimizerIndex) } - async autodetect( - fastaStr: string, - { - onResult, - onError, - onComplete, - }: { - onResult: (r: MinimizerSearchRecord[]) => void - onError?: (error: Error) => void - onComplete?: () => void - }, - ) { + async autodetect(fastaStr: string, { onResult, onError, onComplete }: Callbacks) { this.subscription = this.thread.values().subscribe(onResult, onError, onComplete) await this.thread.autodetect(fastaStr) } diff --git a/packages_rs/nextclade-web/src/state/autodetect.state.ts b/packages_rs/nextclade-web/src/state/autodetect.state.ts index ef392c5d1..5697f4875 100644 --- a/packages_rs/nextclade-web/src/state/autodetect.state.ts +++ b/packages_rs/nextclade-web/src/state/autodetect.state.ts @@ -50,7 +50,7 @@ export const autodetectResultByIndexAtom = selectorFamily { const names = unique(records.flatMap((record) => record.result.datasets.map((dataset) => dataset.name))) let byDataset = {} for (const name of names) { @@ -119,3 +119,15 @@ export const hasAutodetectResultsAtom = selector({ return get(numberAutodetectResultsAtom) > 0 }, }) + +export enum AutodetectRunState { + Idle = 'Idle', + Started = 'Started', + Failed = 'Failed', + Done = 'Done', +} + +export const autodetectRunStateAtom = atom({ + key: 'autodetectRunStateAtom', + default: AutodetectRunState.Idle, +}) From a002c12d2832821fd1dbcb8073ac9855e45b1140 Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Mon, 18 Sep 2023 09:46:56 +0200 Subject: [PATCH 50/51] refactor: lint --- packages_rs/nextclade-web/src/io/AlgorithmInput.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages_rs/nextclade-web/src/io/AlgorithmInput.ts b/packages_rs/nextclade-web/src/io/AlgorithmInput.ts index 5e3d398ad..5ae41c3e5 100644 --- a/packages_rs/nextclade-web/src/io/AlgorithmInput.ts +++ b/packages_rs/nextclade-web/src/io/AlgorithmInput.ts @@ -26,7 +26,7 @@ export class AlgorithmInputFile implements AlgorithmInput { // eslint-disable-next-line unicorn/prefer-ternary if (this.file.webkitRelativePath.trim().length > 0) { - this.path = this.file.webkitRelativePath ?? `${this.uid}-${this.file}` + this.path = this.file.webkitRelativePath } else { this.path = `${this.uid}-${this.file.name}` } From 996980deb570512673c7c4bc5c466c96968c846e Mon Sep 17 00:00:00 2001 From: ivan-aksamentov Date: Mon, 18 Sep 2023 09:49:09 +0200 Subject: [PATCH 51/51] fix(web): text of example entry in query sequence list --- packages_rs/nextclade-web/src/io/AlgorithmInput.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages_rs/nextclade-web/src/io/AlgorithmInput.ts b/packages_rs/nextclade-web/src/io/AlgorithmInput.ts index 5ae41c3e5..ea6e33e8f 100644 --- a/packages_rs/nextclade-web/src/io/AlgorithmInput.ts +++ b/packages_rs/nextclade-web/src/io/AlgorithmInput.ts @@ -103,7 +103,7 @@ export class AlgorithmInputDefault implements AlgorithmInput { constructor(dataset: Dataset) { this.dataset = dataset - this.path = `${this.dataset.path}/${this.dataset.files.examples ?? 'unknown.fasta'}` + this.path = `Examples for '${this.dataset.path}'` } public get name(): string {