Merge branch 'master' into feat/composite-fitness

.cargo/config.toml

@@ -22,6 +22,33 @@ opt-level = 3
 [profile.dev.package.pretty_assertions]
 opt-level = 3
 
+
+[profile.opt-dev]
+inherits = "dev"
+codegen-units = 128
+incremental = true
+opt-level = 2
+lto = "off"
+panic = "abort"
+strip = true
+
+# Optimize dependencies even in debug mode
+[profile.opt-dev.package."*"]
+opt-level = 2
+[profile.opt-dev.package.backtrace]
+opt-level = 3
+[profile.opt-dev.package.color-spantrace]
+opt-level = 3
+[profile.opt-dev.package.tracing]
+opt-level = 3
+[profile.opt-dev.package.tracing-error]
+opt-level = 3
+[profile.opt-dev.package.tracing-subscriber]
+opt-level = 3
+[profile.opt-dev.package.pretty_assertions]
+opt-level = 3
+
+
 [profile.release]
 codegen-units = 1
 incremental = false

CHANGELOG.md

@@ -1,3 +1,14 @@
+## Nextclade Web 2.7.0 (2022-10-05)
+
+### Hide custom clade columns
+
+We added ability to mark certain custom clade columns as hidden. In this case they are not shown in Nextclade Web. This prepares the web application for the upcoming reorganization of clade columns. It should not affect current users.
+
+### Remove unused fields from output files, add custom phenotype key list
+
+We removed extra repetitive fields related to custom phenotype columns (e.g. "Immune escape" and "ACE-2 binding") entries from JSON and NDJSON output files. We also added keys for custom phenotype columns to the header section of output JSON, for symmetry with custom clade columns. These changes should not affect most users.
+
+
 ## Nextclade Web 2.6.1 (2022-09-28)
 
 ### Prettier tooltips

docker-cross

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+# Runs bulk cross-compilation of CLI binaries for all supported platforms
+
 set -euo pipefail
 
 function abspath() {

docker-dev

@@ -1,5 +1,60 @@
 #!/usr/bin/env bash
 
+# Runs various day-to-day dev commands inside dev docker container.
+#
+# The container includes all the tools required for maintaining the project. But this container and script exist
+# for convenience only and are not required for operations. The usual `cargo` and `yarn` workflows can still be used
+# (see dev guide).
+#
+# See the full list of possible commands in the long `case` block halfway through the script.
+# Here are some of the most useful ones:
+#
+# * Build nextclade binary in debug mode:
+#   ./docker-dev b nextclade
+#
+# * Build nextclade binary in release mode:
+#   ./docker-dev br nextclade
+#
+# * Build and run nextclade binary in debug mode, with arguments:
+#   ./docker-dev r nextclade -- dataset list --help
+#
+# * Build and run nextclade binary in release mode, with arguments:
+#   ./docker-dev rr nextclade -- dataset list --help
+#
+# * Cross-compile nextclade binary for Windows in release mode:
+#   CROSS=x86_64-pc-windows-gnu ./docker-dev br
+#
+# * Build WebAssembly module in debug mode:
+#   ./docker-dev W
+#
+# * Build WebAssembly module in release mode:
+#   ./docker-dev Wr
+#
+# * Build and run web application in debug mode:
+#   ./docker-dev a
+#
+# * Build and run web application in release mode:
+#   ./docker-dev war
+#
+# * Lint Rust code:
+#   ./docker-dev l
+#
+# * Lint Rust code, apply automatic fixes:
+#   ./docker-dev lf
+#
+# * Format Rust code:
+#   ./docker-dev f
+#
+# * Shell into the container:
+#   ./docker-dev bash
+#
+# * Run arbitrary command inside the container:
+#   ./docker-dev cargo --version
+#
+# * Run arbitrary yarn command, in the context of web app, inside the container:
+#   ./docker-dev a yarn add --dev react
+#
+
 set -euo pipefail
 
 function abspath() {

docker/docker-dev.dockerfile

@@ -186,6 +186,11 @@ RUN set -euxo pipefail >/dev/null \
 && curl -sSL "https://github.com/rustwasm/wasm-bindgen/releases/download/${WASM_BINDGEN_CLI_VERSION}/wasm-bindgen-${WASM_BINDGEN_CLI_VERSION}-x86_64-unknown-linux-musl.tar.gz" | tar -C "${CARGO_HOME}/bin" --strip-components=1 -xz "wasm-bindgen-${WASM_BINDGEN_CLI_VERSION}-x86_64-unknown-linux-musl/wasm-bindgen" \
 && chmod +x "${CARGO_HOME}/bin/wasm-bindgen"
 
+RUN set -euxo pipefail >/dev/null \
+&& export BINARYEN_VERSION="110" \
+&& curl -sSL "https://github.com/WebAssembly/binaryen/releases/download/version_${BINARYEN_VERSION}/binaryen-version_${BINARYEN_VERSION}-x86_64-linux.tar.gz" | tar -C "${CARGO_HOME}/bin" --strip-components=2 -xz "binaryen-version_${BINARYEN_VERSION}/bin/wasm-opt" \
+&& chmod +x "${CARGO_HOME}/bin/wasm-opt"
+
 # Install executable dependencies
 RUN set -euxo pipefail >/dev/null \
 && cargo quickinstall cargo-audit \

packages_rs/nextclade-cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "nextclade-cli"
-version = "2.6.0"
+version = "2.7.0"
 description = "Alignment, mutation calling, phylogenetic placement, clade assignment and quality control checks for viral genetic sequences. CLI module."
 repository = "https://github.com/nextstrain/nextclade"
 documentation = "https://docs.nextstrain.org/projects/nextclade/en/stable/"

packages_rs/nextclade-cli/src/cli/nextclade_loop.rs

@@ -10,7 +10,7 @@ use itertools::Itertools;
 use log::info;
 use nextclade::align::gap_open::{get_gap_open_close_scores_codon_aware, get_gap_open_close_scores_flat};
 use nextclade::align::params::AlignPairwiseParams;
-use nextclade::analyze::phenotype::get_phenotype_attr_keys;
+use nextclade::analyze::phenotype::get_phenotype_attr_descs;
 use nextclade::io::fasta::{FastaReader, FastaRecord};
 use nextclade::io::fs::has_extension;
 use nextclade::io::json::json_write;
@@ -130,7 +130,7 @@ pub fn nextclade_run(run_args: NextcladeRunArgs) -> Result<(), Report> {
   let should_keep_outputs = output_tree.is_some();
   let mut outputs = Vec::<NextcladeOutputs>::new();
 
-  let phenotype_attr_keys = get_phenotype_attr_keys(&virus_properties);
+  let phenotype_attrs = &get_phenotype_attr_descs(&virus_properties);
 
   std::thread::scope(|s| {
     const CHANNEL_SIZE: usize = 128;
@@ -223,7 +223,7 @@ pub fn nextclade_run(run_args: NextcladeRunArgs) -> Result<(), Report> {
       let mut output_writer = NextcladeOrderedWriter::new(
         gene_map,
         clade_node_attrs,
-        &phenotype_attr_keys,
+        phenotype_attrs,
         &output_fasta,
         &output_json,
         &output_ndjson,

packages_rs/nextclade-cli/src/cli/nextclade_ordered_writer.rs

@@ -2,6 +2,7 @@ use crate::cli::nextclade_loop::NextcladeRecord;
 use eyre::{Report, WrapErr};
 use itertools::Itertools;
 use log::warn;
+use nextclade::analyze::virus_properties::PhenotypeAttrDesc;
 use nextclade::io::errors_csv::ErrorsCsvWriter;
 use nextclade::io::fasta::{FastaPeptideWriter, FastaRecord, FastaWriter};
 use nextclade::io::gene_map::GeneMap;
@@ -38,7 +39,7 @@ impl<'a> NextcladeOrderedWriter<'a> {
   pub fn new(
     gene_map: &'a GeneMap,
     clade_node_attr_key_descs: &[CladeNodeAttrKeyDesc],
-    phenotype_attr_keys: &[String],
+    phenotype_attr_key_desc: &[PhenotypeAttrDesc],
     output_fasta: &Option<PathBuf>,
     output_json: &Option<PathBuf>,
     output_ndjson: &Option<PathBuf>,
@@ -59,8 +60,9 @@ impl<'a> NextcladeOrderedWriter<'a> {
     let errors_csv_writer =
       output_errors.map_ref_fallible(|output_errors| ErrorsCsvWriter::new(gene_map, &output_errors))?;
 
-    let output_json_writer =
-      output_json.map_ref_fallible(|output_json| ResultsJsonWriter::new(&output_json, clade_node_attr_key_descs))?;
+    let output_json_writer = output_json.map_ref_fallible(|output_json| {
+      ResultsJsonWriter::new(&output_json, clade_node_attr_key_descs, phenotype_attr_key_desc)
+    })?;
 
     let output_ndjson_writer = output_ndjson.map_ref_fallible(NdjsonFileWriter::new)?;
 
@@ -69,12 +71,17 @@ impl<'a> NextcladeOrderedWriter<'a> {
       .map(|desc| desc.name.clone())
       .collect_vec();
 
+    let phenotype_attr_keys = phenotype_attr_key_desc
+      .iter()
+      .map(|desc| desc.name.clone())
+      .collect_vec();
+
     let output_csv_writer = output_csv.map_ref_fallible(|output_csv| {
-      NextcladeResultsCsvFileWriter::new(&output_csv, b';', &clade_node_attr_keys, phenotype_attr_keys)
+      NextcladeResultsCsvFileWriter::new(&output_csv, b';', &clade_node_attr_keys, &phenotype_attr_keys)
     })?;
 
     let output_tsv_writer = output_tsv.map_ref_fallible(|output_tsv| {
-      NextcladeResultsCsvFileWriter::new(&output_tsv, b'\t', &clade_node_attr_keys, phenotype_attr_keys)
+      NextcladeResultsCsvFileWriter::new(&output_tsv, b'\t', &clade_node_attr_keys, &phenotype_attr_keys)
     })?;
 
     Ok(Self {

packages_rs/nextclade-web/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "nextclade-web"
-version = "2.6.0"
+version = "2.7.0"
 description = "Alignment, mutation calling, phylogenetic placement, clade assignment and quality control checks for viral genetic sequences. WebAssembly module."
 edition = "2021"
 license = "MIT"

packages_rs/nextclade-web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nextstrain/nextclade-web",
-  "version": "2.6.1",
+  "version": "2.7.0",
   "description": "Clade assignment, mutation calling, and sequence quality checks",
   "homepage": "https://clades.nextstrain.org",
   "repository": {
@@ -78,7 +78,7 @@
     "monkey-patch": "bash tools/auspice-monkey-patch.sh && yarn run:node tools/monkeyPatch.ts",
     "changelog": "yarn conventional-changelog -p angular -i CHANGELOG.md -s -r 1",
     "serve-data": "babel-node --config-file \"./babel-node.config.js\" --extensions \".ts\" ./tools/server/dataServer.ts",
-    "wasm-dev": "wasm-pack --quiet --log-level=error build --dev --target=bundler --out-dir=.build/wasm-dev/ --out-name=nextclade-wasm -- --package=nextclade-web --locked && mkdir -p src/gen/ && cp .build/wasm-dev/* src/gen/",
+    "wasm-dev": "cargo build -q --profile=opt-dev --package=nextclade-web --lib --target=wasm32-unknown-unknown --target-dir=.build/docker/wasm-dev/ && wasm-bindgen .build/docker/wasm-dev/wasm32-unknown-unknown/opt-dev/nextclade_web.wasm --target=bundler --out-dir=.build/wasm-prod/ --out-name=nextclade-wasm && wasm-opt -O --optimize-level=1 --shrink-level=1 .build/docker/wasm-dev/wasm32-unknown-unknown/opt-dev/nextclade_web.wasm -o .build/wasm-prod/nextclade-wasm.wasm && mkdir -p src/gen/ && cp .build/wasm-prod/* src/gen/",
     "wasm-prod": "wasm-pack --quiet --log-level=error build --release --target=bundler --out-dir=.build/wasm-prod/ --out-name=nextclade-wasm -- --package=nextclade-web --locked && mkdir -p src/gen/ && cp .build/wasm-prod/* src/gen/"
   },
   "dependencies": {

packages_rs/nextclade-web/src/components/Results/ResultsTable.tsx

@@ -16,9 +16,7 @@ import {
 } from 'src/state/settings.state'
 import {
   cladeNodeAttrDescsAtom,
-  cladeNodeAttrKeysAtom,
   phenotypeAttrDescsAtom,
-  phenotypeAttrKeysAtom,
   seqIndicesFilteredAtom,
   sortAnalysisResultsAtom,
   sortAnalysisResultsByKeyAtom,
@@ -69,9 +67,7 @@ export function ResultsTable() {
   const columnWidthsPx = useRecoilValue(resultsTableColumnWidthsPxAtom)
   const dynamicCladeColumnWidthPx = useRecoilValue(resultsTableDynamicCladeColumnWidthPxAtom)
   const dynamicPhenotypeColumnWidthPx = useRecoilValue(resultsTableDynamicPhenotypeColumnWidthPxAtom)
-  const cladeNodeAttrKeys = useRecoilValue(cladeNodeAttrKeysAtom)
   const cladeNodeAttrDescs = useRecoilValue(cladeNodeAttrDescsAtom)
-  const phenotypeAttrKeys = useRecoilValue(phenotypeAttrKeysAtom)
   const phenotypeAttrDescs = useRecoilValue(phenotypeAttrDescsAtom)
 
   const isResultsFilterPanelCollapsed = useRecoilValue(isResultsFilterPanelCollapsedAtom)
@@ -84,15 +80,15 @@ export function ResultsTable() {
       columnWidthsPx,
       dynamicCladeColumnWidthPx,
       dynamicPhenotypeColumnWidthPx,
-      cladeNodeAttrKeys,
-      phenotypeAttrKeys,
+      cladeNodeAttrDescs,
+      phenotypeAttrDescs,
     }))
   }, [
-    cladeNodeAttrKeys,
+    cladeNodeAttrDescs,
     columnWidthsPx,
     dynamicCladeColumnWidthPx,
     dynamicPhenotypeColumnWidthPx,
-    phenotypeAttrKeys,
+    phenotypeAttrDescs,
     seqIndices,
     viewedGene,
   ])
@@ -176,22 +172,24 @@ export function ResultsTable() {
   }, []) // prettier-ignore
 
   const dynamicCladeColumns = useMemo(() => {
-    return cladeNodeAttrDescs.map(({ name: attrKey, displayName, description }) => {
-      const sortAsc = sortByKey(attrKey, SortDirection.asc)
-      const sortDesc = sortByKey(attrKey, SortDirection.desc)
-      return (
-        <TableHeaderCell key={attrKey} basis={dynamicCladeColumnWidthPx} grow={0} shrink={0}>
-          <TableHeaderCellContent>
-            <TableCellText>{displayName}</TableCellText>
-            <ResultsControlsSort sortAsc={sortAsc} sortDesc={sortDesc} />
-          </TableHeaderCellContent>
-          <ButtonHelpStyled identifier={`btn-help-col-clade-${attrKey}`} tooltipWidth="600px">
-            <h5>{`Column: ${displayName}`}</h5>
-            <p>{description}</p>
-          </ButtonHelpStyled>
-        </TableHeaderCell>
-      )
-    })
+    return cladeNodeAttrDescs
+      .filter((attr) => !attr.hideInWeb)
+      .map(({ name: attrKey, displayName, description }) => {
+        const sortAsc = sortByKey(attrKey, SortDirection.asc)
+        const sortDesc = sortByKey(attrKey, SortDirection.desc)
+        return (
+          <TableHeaderCell key={attrKey} basis={dynamicCladeColumnWidthPx} grow={0} shrink={0}>
+            <TableHeaderCellContent>
+              <TableCellText>{displayName}</TableCellText>
+              <ResultsControlsSort sortAsc={sortAsc} sortDesc={sortDesc} />
+            </TableHeaderCellContent>
+            <ButtonHelpStyled identifier={`btn-help-col-clade-${attrKey}`} tooltipWidth="600px">
+              <h5>{`Column: ${displayName}`}</h5>
+              <p>{description}</p>
+            </ButtonHelpStyled>
+          </TableHeaderCell>
+        )
+      })
   }, [cladeNodeAttrDescs, dynamicCladeColumnWidthPx, sortByKey])
 
   const dynamicPhenotypeColumns = useMemo(() => {