-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Add config setting * WIP * Implement something * Add caching * Support the field in values.yaml * default to false * Be explicit about using ISO format * test enabling it for west-nile * Add documentation * Add metadata field * generate default * Add it for Ebola Sudan * Config fix? * Parse only date * Now fix? * Update docs * Catch IllegalArgumentException and log and ignore * Validate settings file * Write test to check for sorted * Move test * Factor out finder into a utility * Simplify; document * Add test stub * Add 'createDummyRow' function for testing * Add EarliestDateFinder Test * Add comment to new test for ordering * Add assertion * format * explain the field * explain the field * Add initial empty test * Add positive test * Add more tests * make code more compact * Update backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataEndpointTest.kt Co-authored-by: Chaoran Chen <[email protected]> * Make test naming more specific * Improve error emssage and raise log level --------- Co-authored-by: Chaoran Chen <[email protected]>
- Loading branch information
1 parent
706d11a
commit 43ca5e8
Showing
10 changed files
with
388 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
74 changes: 74 additions & 0 deletions
74
backend/src/main/kotlin/org/loculus/backend/utils/EarliestReleaseDateFinder.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| package org.loculus.backend.utils | ||
|
|
||
| import kotlinx.datetime.LocalDate | ||
| import kotlinx.datetime.LocalDateTime | ||
| import kotlinx.datetime.LocalTime | ||
| import mu.KotlinLogging | ||
| import org.loculus.backend.service.submission.RawProcessedData | ||
|
|
||
| private val log = KotlinLogging.logger { } | ||
|
|
||
| /** | ||
| * Calculate the earliest release date for rows of sequence entries given to it one by one. | ||
| * Assumes that rows are sorted: all accession entries are given in a block, and with ascending versions. | ||
| * | ||
| * The earliest release date of a sequence is the earliest date of: | ||
| * - the internal release date | ||
| * - any date from a given list of fields | ||
| * - the earliest release date from the previous version (if it exists) | ||
| */ | ||
| class EarliestReleaseDateFinder(private val fields: List<String>) { | ||
| private val earliestReleaseDateCache = mutableMapOf<String, LocalDateTime>() | ||
| private var previousRawProcessedData: RawProcessedData? = null | ||
|
|
||
| fun calculateEarliestReleaseDate(rawProcessedData: RawProcessedData): LocalDateTime { | ||
| assert( | ||
| previousRawProcessedData == null || | ||
| rawProcessedData.accession > previousRawProcessedData!!.accession || | ||
| ( | ||
| rawProcessedData.accession == previousRawProcessedData!!.accession && | ||
| rawProcessedData.version > previousRawProcessedData!!.version | ||
| ), | ||
| ) { | ||
| "Input is not ordered. Current: ${rawProcessedData.accession}.${rawProcessedData.version}, " + | ||
| "Previous: ${previousRawProcessedData!!.accession}.${previousRawProcessedData!!.version}" | ||
| } | ||
|
|
||
| var earliestReleaseDate = rawProcessedData.releasedAtTimestamp | ||
|
|
||
| fields.forEach { field -> | ||
| rawProcessedData.processedData.metadata[field]?.textValue()?.let { dateText -> | ||
| val date = try { | ||
| LocalDateTime(LocalDate.parse(dateText), LocalTime.fromSecondOfDay(0)) | ||
| } catch (e: IllegalArgumentException) { | ||
| log.error { | ||
| "Unexpected error: Incorrectly formatted date on ${rawProcessedData.accession}." + | ||
| "${rawProcessedData.version} on field $field: $dateText " + | ||
| "Something is wrong with this instance: it might be a configuration error or a bug of " + | ||
| "the software. Please feel free to reach out to the developers if you need advice " + | ||
| "(https://github.com/loculus-project/loculus/issues/)." | ||
| } | ||
| null | ||
| } | ||
| if (date != null) { | ||
| earliestReleaseDate = if (date < earliestReleaseDate) date else earliestReleaseDate | ||
| } | ||
| } | ||
| } | ||
|
|
||
| earliestReleaseDateCache[rawProcessedData.accession]?.let { cached -> | ||
| if (cached < earliestReleaseDate) { | ||
| earliestReleaseDate = cached | ||
| } else { | ||
| earliestReleaseDateCache[rawProcessedData.accession] = earliestReleaseDate | ||
| } | ||
| } ?: run { | ||
| earliestReleaseDateCache.clear() // Inputs are ordered; no need for previous values | ||
| earliestReleaseDateCache[rawProcessedData.accession] = earliestReleaseDate | ||
| } | ||
|
|
||
| previousRawProcessedData = rawProcessedData | ||
|
|
||
| return earliestReleaseDate | ||
| } | ||
| } |
74 changes: 74 additions & 0 deletions
74
backend/src/test/kotlin/org/loculus/backend/config/BackendSpringConfigTest.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| package org.loculus.backend.config | ||
|
|
||
| import org.hamcrest.MatcherAssert.assertThat | ||
| import org.hamcrest.Matchers.`is` | ||
| import org.junit.jupiter.api.Assertions.assertTrue | ||
| import org.junit.jupiter.api.Test | ||
| import org.loculus.backend.controller.DEFAULT_ORGANISM | ||
|
|
||
| class BackendSpringConfigTest { | ||
|
|
||
| @Test | ||
| fun `GIVEN an empty config THEN the it is valid`() { | ||
| val conf = backendConfig(emptyList(), EarliestReleaseDate(false, emptyList())) | ||
|
|
||
| val errors = validateEarliestReleaseDateFields(conf) | ||
|
|
||
| assertTrue(errors.isEmpty()) | ||
| } | ||
|
|
||
| @Test | ||
| fun `GIVEN a config with earliestReleaseDate configured with existing date fields THEN it is valid`() { | ||
| val conf = backendConfig( | ||
| listOf( | ||
| Metadata("foo", MetadataType.DATE), | ||
| Metadata("bar", MetadataType.DATE), | ||
| ), | ||
| EarliestReleaseDate(true, listOf("foo", "bar")), | ||
| ) | ||
|
|
||
| val errors = validateEarliestReleaseDateFields(conf) | ||
|
|
||
| assertTrue(errors.isEmpty()) | ||
| } | ||
|
|
||
| @Test | ||
| fun `GIVEN a config with a missing external field in earliestReleaseDate THEN it is invalid`() { | ||
| val conf = backendConfig( | ||
| listOf( | ||
| Metadata("foo", MetadataType.DATE), | ||
| ), | ||
| EarliestReleaseDate(true, listOf("foo", "bar")), | ||
| ) | ||
|
|
||
| val errors = validateEarliestReleaseDateFields(conf) | ||
|
|
||
| assertThat(errors.size, `is`(1)) | ||
| } | ||
|
|
||
| @Test | ||
| fun `GIVEN a config with an external field with incorrect type in earliestReleaseDate THEN it is invalid`() { | ||
| val conf = backendConfig( | ||
| listOf( | ||
| Metadata("foo", MetadataType.DATE), | ||
| Metadata("bar", MetadataType.STRING), | ||
| ), | ||
| EarliestReleaseDate(true, listOf("foo", "bar")), | ||
| ) | ||
|
|
||
| val errors = validateEarliestReleaseDateFields(conf) | ||
|
|
||
| assertThat(errors.size, `is`(1)) | ||
| } | ||
| } | ||
|
|
||
| fun backendConfig(metadataList: List<Metadata>, earliestReleaseDate: EarliestReleaseDate) = BackendConfig( | ||
| organisms = mapOf( | ||
| DEFAULT_ORGANISM to InstanceConfig( | ||
| Schema(DEFAULT_ORGANISM, metadataList, earliestReleaseDate = earliestReleaseDate), | ||
| ReferenceGenome(emptyList(), emptyList()), | ||
| ), | ||
| ), | ||
| accessionPrefix = "FOO_", | ||
| dataUseTermsUrls = null, | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.