Merge pull request #6 from nextstrain/ingest-curate-fixes

Ingest curate fixes

ingest/config/defaults.yaml

@@ -22,10 +22,27 @@ curate:
   # The path to the local geolocation rules within the pathogen repo
   # The path should be relative to the ingest directory.
   local_geolocation_rules: "config/geolocation_rules.tsv"
-  # List of field names to change in the format of <old_field_name>=<new_field_name>
+  # List of field names to change where the key is the original field name and the value is the new field name
   # This is the first step in the pipeline, so any references to field names
   # in the configs below should use the new field names
-  field_map: []
+  # The examples below are based on the NCBI Datasets output TSV column names, your data might have different field names.
+  field_map:
+    Source database: database
+    Isolate Collection date: date
+    Release date: date_released
+    Update date: date_updated
+    Accession: accession
+    Isolate Lineage: strain
+    Geographic Region: region
+    Geographic Location: location
+    Submitter Names: authors
+    Submitter Affiliation: institution
+    SRA Accessions: sra_accessions
+    Length: length
+    Host Name: host
+    Isolate Lineage source: sample_type
+    BioSample accession: biosample_accession
+    Submitter Country: submitter_country
   # List of date fields to standardize to ISO format YYYY-MM-DD
   date_fields: []
   # List of expected date formats that are present in the date fields provided above

ingest/rules/curate.smk

@@ -34,6 +34,13 @@ rule concat_geolocation_rules:
         """
 
 
+def format_field_map(field_map: dict[str, str]) -> str:
+    """
+    Format dict to `"key1"="value1" "key2"="value2"...` for use in shell commands.
+    """
+    return " ".join([f'"{key}"="{value}"' for key, value in field_map.items()])
+
+
 # This curate pipeline is based on existing pipelines for pathogen repos using NCBI data.
 # You may want to add and/or remove steps from the pipeline for custom metadata
 # curation for your pathogen. Note that the curate pipeline is streaming NDJSON
@@ -55,7 +62,7 @@ rule curate:
     benchmark:
         "benchmarks/curate.txt"
     params:
-        field_map=config["curate"]["field_map"],
+        field_map=format_field_map(config["curate"]["field_map"]),
         date_fields=config["curate"]["date_fields"],
         expected_date_formats=config["curate"]["expected_date_formats"],
         articles=config["curate"]["titlecase"]["articles"],
@@ -104,7 +111,7 @@ rule subset_metadata:
     output:
         subset_metadata="results/subset_metadata.tsv",
     params:
-        metadata_fields=config["curate"]["metadata_columns"],
+        metadata_fields=",".join(config["curate"]["metadata_columns"]),
     shell:
         """
         tsv-select -H -f {params.metadata_fields} \