From 95e9ac81262dc78713cd681edcc2200d263974e1 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Mon, 2 Oct 2023 17:35:06 -0400 Subject: [PATCH 01/12] Add spy for pmidLookup --- .../loader/nihms/TransformAndLoadSmokeIT.java | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/pass-nihms-loader/nihms-data-transform-load/src/test/java/org/eclipse/pass/loader/nihms/TransformAndLoadSmokeIT.java b/pass-nihms-loader/nihms-data-transform-load/src/test/java/org/eclipse/pass/loader/nihms/TransformAndLoadSmokeIT.java index 38b4a069..4eca72a5 100644 --- a/pass-nihms-loader/nihms-data-transform-load/src/test/java/org/eclipse/pass/loader/nihms/TransformAndLoadSmokeIT.java +++ b/pass-nihms-loader/nihms-data-transform-load/src/test/java/org/eclipse/pass/loader/nihms/TransformAndLoadSmokeIT.java @@ -3,10 +3,14 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.when; import java.io.File; import java.util.List; +import org.eclipse.pass.entrez.PmidLookup; import org.eclipse.pass.support.client.PassClientSelector; import org.eclipse.pass.support.client.RSQL; import org.eclipse.pass.support.client.model.Grant; @@ -16,14 +20,22 @@ import org.eclipse.pass.support.client.model.User; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; /** * Smoke tests loads in some test data from spreadsheets and verifies it all loaded in as expected * * @author Karen Hanson */ +@ExtendWith(MockitoExtension.class) public class TransformAndLoadSmokeIT extends NihmsSubmissionEtlITBase { + @Mock + private PmidLookup pmidLookup; + @BeforeEach public void setup() throws Exception { preLoadGrants(); @@ -37,9 +49,16 @@ public void setup() throws Exception { */ @Test public void smokeTestLoadAndTransform() throws Exception { - NihmsTransformLoadApp app = new NihmsTransformLoadApp(null); - app.run(); + NihmsTransformLoadApp spyApp = Mockito.spy(app); + + // mocking the entrez pmid lookup. it will not return an entrez record because these are fake grants. + // this will save runtime + //when(pmidLookup.retrievePubMedRecord(any())).thenReturn(null); + doReturn(null).when(pmidLookup).retrievePubMedRecord(any()); + + spyApp.run(); + PassClientSelector repoCopySelector = new PassClientSelector<>(RepositoryCopy.class); PassClientSelector publicationSelector = new PassClientSelector<>(Publication.class); PassClientSelector submissionSelector = new PassClientSelector<>(Submission.class); From 3fb97cae1119e0cbecf92c99fbed0b788a681547 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Tue, 3 Oct 2023 13:31:35 -0400 Subject: [PATCH 02/12] Add ENTREZ_TIME_OUT --- .../java/org/eclipse/pass/entrez/PmidLookup.java | 6 +++++- .../loader/nihms/TransformAndLoadSmokeIT.java | 16 +++------------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/pass-nihms-loader/entrez-pmid-lookup/src/main/java/org/eclipse/pass/entrez/PmidLookup.java b/pass-nihms-loader/entrez-pmid-lookup/src/main/java/org/eclipse/pass/entrez/PmidLookup.java index 6af29c21..08f6d8db 100644 --- a/pass-nihms-loader/entrez-pmid-lookup/src/main/java/org/eclipse/pass/entrez/PmidLookup.java +++ b/pass-nihms-loader/entrez-pmid-lookup/src/main/java/org/eclipse/pass/entrez/PmidLookup.java @@ -56,17 +56,21 @@ public class PmidLookup { ".fcgi?db=pubmed&retmode=json&rettype=abstract&id=%s"; private static final String ENTREZ_PATH_KEY = "entrez.pmid.path"; + private static final String ENTREZ_TIME_OUT_KEY = "entrez.time.out"; + private static final String ENTREZ_TIME_OUT = "400"; private static final String JSON_ERROR_KEY = "error"; private static final String JSON_RESULT_KEY = "result"; private String entrezPath; + private String entrezTimeout; /** * Default constructor uses the default Entrez path */ public PmidLookup() { entrezPath = System.getProperty(ENTREZ_PATH_KEY, DEFAULT_ENTREZ_PATH); + entrezTimeout = System.getProperty(ENTREZ_TIME_OUT_KEY, ENTREZ_TIME_OUT); } /** @@ -101,7 +105,7 @@ public JSONObject retrievePubMedRecordAsJson(String pmid) { if (jsonRecord == null) { // pause and retry once to allow for API limitations LOG.info("Pausing before trying to pull PMID {} from Entrez again", pmid); - TimeUnit.MILLISECONDS.sleep(400); + TimeUnit.MILLISECONDS.sleep(Long.parseLong(entrezTimeout)); jsonRecord = retrieveJsonFromApi(pmid); } } catch (InterruptedException e) { diff --git a/pass-nihms-loader/nihms-data-transform-load/src/test/java/org/eclipse/pass/loader/nihms/TransformAndLoadSmokeIT.java b/pass-nihms-loader/nihms-data-transform-load/src/test/java/org/eclipse/pass/loader/nihms/TransformAndLoadSmokeIT.java index 4eca72a5..949b033e 100644 --- a/pass-nihms-loader/nihms-data-transform-load/src/test/java/org/eclipse/pass/loader/nihms/TransformAndLoadSmokeIT.java +++ b/pass-nihms-loader/nihms-data-transform-load/src/test/java/org/eclipse/pass/loader/nihms/TransformAndLoadSmokeIT.java @@ -5,7 +5,6 @@ import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.when; import java.io.File; import java.util.List; @@ -21,8 +20,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; import org.mockito.Mockito; +import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; /** @@ -33,9 +32,6 @@ @ExtendWith(MockitoExtension.class) public class TransformAndLoadSmokeIT extends NihmsSubmissionEtlITBase { - @Mock - private PmidLookup pmidLookup; - @BeforeEach public void setup() throws Exception { preLoadGrants(); @@ -49,22 +45,16 @@ public void setup() throws Exception { */ @Test public void smokeTestLoadAndTransform() throws Exception { - NihmsTransformLoadApp app = new NihmsTransformLoadApp(null); - NihmsTransformLoadApp spyApp = Mockito.spy(app); - // mocking the entrez pmid lookup. it will not return an entrez record because these are fake grants. - // this will save runtime - //when(pmidLookup.retrievePubMedRecord(any())).thenReturn(null); - doReturn(null).when(pmidLookup).retrievePubMedRecord(any()); + NihmsTransformLoadApp app = new NihmsTransformLoadApp(null); - spyApp.run(); + app.run(); PassClientSelector repoCopySelector = new PassClientSelector<>(RepositoryCopy.class); PassClientSelector publicationSelector = new PassClientSelector<>(Publication.class); PassClientSelector submissionSelector = new PassClientSelector<>(Submission.class); //now that it has run lets do some basic tallys to make sure they are as expected: - //make sure RepositoryCopies are all in before moving on so we can be sure the counts are done. repoCopySelector.setFilter(RSQL.notEquals("id", "-1")); List repositoryCopies = passClient.selectObjects(repoCopySelector).getObjects(); From c88a19049555662a7b1b460d9c8cd84966d08709 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Tue, 3 Oct 2023 17:16:13 -0400 Subject: [PATCH 03/12] Remove spy --- .../java/org/eclipse/pass/entrez/PmidLookup.java | 4 ++-- .../nihms-data-transform-load/pom.xml | 16 ++++++++++++---- .../loader/nihms/TransformAndLoadSmokeIT.java | 11 ++--------- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/pass-nihms-loader/entrez-pmid-lookup/src/main/java/org/eclipse/pass/entrez/PmidLookup.java b/pass-nihms-loader/entrez-pmid-lookup/src/main/java/org/eclipse/pass/entrez/PmidLookup.java index 08f6d8db..2dbb069d 100644 --- a/pass-nihms-loader/entrez-pmid-lookup/src/main/java/org/eclipse/pass/entrez/PmidLookup.java +++ b/pass-nihms-loader/entrez-pmid-lookup/src/main/java/org/eclipse/pass/entrez/PmidLookup.java @@ -57,7 +57,7 @@ public class PmidLookup { private static final String ENTREZ_PATH_KEY = "entrez.pmid.path"; private static final String ENTREZ_TIME_OUT_KEY = "entrez.time.out"; - private static final String ENTREZ_TIME_OUT = "400"; + private static final String DEFAULT_ENTREZ_TIME_OUT = "400"; private static final String JSON_ERROR_KEY = "error"; private static final String JSON_RESULT_KEY = "result"; @@ -70,7 +70,7 @@ public class PmidLookup { */ public PmidLookup() { entrezPath = System.getProperty(ENTREZ_PATH_KEY, DEFAULT_ENTREZ_PATH); - entrezTimeout = System.getProperty(ENTREZ_TIME_OUT_KEY, ENTREZ_TIME_OUT); + entrezTimeout = System.getProperty(ENTREZ_TIME_OUT_KEY, DEFAULT_ENTREZ_TIME_OUT); } /** diff --git a/pass-nihms-loader/nihms-data-transform-load/pom.xml b/pass-nihms-loader/nihms-data-transform-load/pom.xml index 2b378b41..fa02b6e1 100644 --- a/pass-nihms-loader/nihms-data-transform-load/pom.xml +++ b/pass-nihms-loader/nihms-data-transform-load/pom.xml @@ -70,6 +70,13 @@ logback-classic + + org.apache.commons + commons-collections4 + 4.2 + compile + + org.mockito @@ -84,11 +91,12 @@ 4.8.1 test + - org.apache.commons - commons-collections4 - 4.2 - compile + com.github.tomakehurst + wiremock-jre8 + 2.35.0 + test diff --git a/pass-nihms-loader/nihms-data-transform-load/src/test/java/org/eclipse/pass/loader/nihms/TransformAndLoadSmokeIT.java b/pass-nihms-loader/nihms-data-transform-load/src/test/java/org/eclipse/pass/loader/nihms/TransformAndLoadSmokeIT.java index 949b033e..38b0c3dd 100644 --- a/pass-nihms-loader/nihms-data-transform-load/src/test/java/org/eclipse/pass/loader/nihms/TransformAndLoadSmokeIT.java +++ b/pass-nihms-loader/nihms-data-transform-load/src/test/java/org/eclipse/pass/loader/nihms/TransformAndLoadSmokeIT.java @@ -3,13 +3,11 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doReturn; import java.io.File; import java.util.List; -import org.eclipse.pass.entrez.PmidLookup; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; import org.eclipse.pass.support.client.PassClientSelector; import org.eclipse.pass.support.client.RSQL; import org.eclipse.pass.support.client.model.Grant; @@ -19,17 +17,13 @@ import org.eclipse.pass.support.client.model.User; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mockito; -import org.mockito.Spy; -import org.mockito.junit.jupiter.MockitoExtension; /** * Smoke tests loads in some test data from spreadsheets and verifies it all loaded in as expected * * @author Karen Hanson */ -@ExtendWith(MockitoExtension.class) +@WireMockTest public class TransformAndLoadSmokeIT extends NihmsSubmissionEtlITBase { @BeforeEach @@ -45,7 +39,6 @@ public void setup() throws Exception { */ @Test public void smokeTestLoadAndTransform() throws Exception { - NihmsTransformLoadApp app = new NihmsTransformLoadApp(null); app.run(); From b8aa610ddd64b9442ef40b037fd718d68cdcc9dd Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Wed, 4 Oct 2023 17:59:17 -0400 Subject: [PATCH 04/12] Update WireMock --- .../nihms-data-transform-load/pom.xml | 6 +++--- .../loader/nihms/TransformAndLoadSmokeIT.java | 16 +++++++++++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/pass-nihms-loader/nihms-data-transform-load/pom.xml b/pass-nihms-loader/nihms-data-transform-load/pom.xml index fa02b6e1..be0a11d4 100644 --- a/pass-nihms-loader/nihms-data-transform-load/pom.xml +++ b/pass-nihms-loader/nihms-data-transform-load/pom.xml @@ -93,9 +93,9 @@ - com.github.tomakehurst - wiremock-jre8 - 2.35.0 + org.wiremock + wiremock + 3.2.0 test diff --git a/pass-nihms-loader/nihms-data-transform-load/src/test/java/org/eclipse/pass/loader/nihms/TransformAndLoadSmokeIT.java b/pass-nihms-loader/nihms-data-transform-load/src/test/java/org/eclipse/pass/loader/nihms/TransformAndLoadSmokeIT.java index 38b0c3dd..93db823a 100644 --- a/pass-nihms-loader/nihms-data-transform-load/src/test/java/org/eclipse/pass/loader/nihms/TransformAndLoadSmokeIT.java +++ b/pass-nihms-loader/nihms-data-transform-load/src/test/java/org/eclipse/pass/loader/nihms/TransformAndLoadSmokeIT.java @@ -1,5 +1,9 @@ package org.eclipse.pass.loader.nihms; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.ok; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlMatching; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.fail; @@ -7,6 +11,7 @@ import java.io.File; import java.util.List; +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; import com.github.tomakehurst.wiremock.junit5.WireMockTest; import org.eclipse.pass.support.client.PassClientSelector; import org.eclipse.pass.support.client.RSQL; @@ -38,7 +43,16 @@ public void setup() throws Exception { * @throws Exception if an error occurs */ @Test - public void smokeTestLoadAndTransform() throws Exception { + public void smokeTestLoadAndTransform(WireMockRuntimeInfo wmRuntimeInfo) throws Exception { + final int wmPort = wmRuntimeInfo.getHttpPort(); + System.setProperty("entrez.pmid.path", "http://localhost:" + wmPort + + "/entrez/eutils/esummary.fcgi?db=pubmed&retmode=json&rettype=abstract&id=%s"); + System.setProperty("entrez.time.out", "0"); + String jsonErrorResponse = "{\"error\": \"cannot get document summary\"}"; + + stubFor(get(urlMatching("/entrez/eutils/esummary.fcgi\\?db=pubmed&retmode=json&rettype=abstract&id=([0-9]*)")) + .willReturn(ok(jsonErrorResponse))); + NihmsTransformLoadApp app = new NihmsTransformLoadApp(null); app.run(); From 1710fbce3a720b9d4b1315ab425447b0bdb9e511 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Fri, 6 Oct 2023 16:56:35 -0400 Subject: [PATCH 05/12] Remove similar award numbers --- .../test/resources/valid_award_numbers.csv | 101 +----------------- 1 file changed, 1 insertion(+), 100 deletions(-) diff --git a/pass-nihms-loader/nihms-pass-client/src/test/resources/valid_award_numbers.csv b/pass-nihms-loader/nihms-pass-client/src/test/resources/valid_award_numbers.csv index b1d5a450..f054a5c0 100644 --- a/pass-nihms-loader/nihms-pass-client/src/test/resources/valid_award_numbers.csv +++ b/pass-nihms-loader/nihms-pass-client/src/test/resources/valid_award_numbers.csv @@ -5,8 +5,6 @@ M-P1110-07 10270932 RSR13R016 FA8750-19-2-0034 -90062452 -070119 1R03DA026905-01 D4880C00003 K24HD052559 @@ -30,8 +28,6 @@ BR4380 N00014-16-1-2406 R01AI24576 19PRE34380815 -90055187 -90059165 U18TR003780 ATM0101064 JHU IO#90056635 @@ -55,7 +51,6 @@ OF-50 NNG06GF23G 19-028-ASP A Peer-Based Benchma -90091103 TV5600-CNS-20007 SOW#BN40423 13042486 Winner , La @@ -65,19 +60,13 @@ W81XWH-17-1-0630 BresslerGenentechISA T32AG000120 90072100 -16779 W81XWH-14-1-0396 -0017631 90092667 NA 00038389 -1505621 RFA-MD-11-1 801789 DEC-46 -90081781 200168785 4391CPFF0702 -90032558 -36639 R43HG008709 Project #0117-003 (J 90071038 @@ -92,20 +81,12 @@ R21GM146105 INV-007197 (1063942) JHU IO#90050185 N000140910867 -348-2084 INV-060110 90088694 -90060250 -801786 -37890 -90080644 U01CK000554 NAG511034 -90028708 -90042642 Sub # 1500181 20120865 -110080 R01GM079239 CC5013-MDS-002 FY18-CO#38391 @@ -139,7 +120,6 @@ U01HL096812 304-2279 21-06093 A03-2466 -20899 K99AI151094 90085073 2014-MSCRFI-0715 @@ -147,28 +127,23 @@ R01NS117604 R21HD060169 R36AG083325 CP 12663 2 23 A25 -18154 0630223N U1QHP28710 12-049 PO # 20224722 -63519 S07RR18230 VX09-222-103 IO# 90041542 DHMH-OPASS-17-17108 80NSSC19K0589 PGP0007 2012 -7772 HDTRA11110063 5106332 3000037037 -35833 ZEN062 VAWA-2010-1641 UCHC6-55818869 R01NS048206 -201767 0314-012 12 (KANG) IM101-226-111 ADU-CL-03 @@ -186,13 +161,10 @@ R01CA222779 5-58853 Differentiation of H 3000967623 -1257405 CA209-374 GE FOUNDATION R21NS093244 DEFG0203ER41271 -90072752 -K23AT002862 P021A170035 IPN 23087127 886-2120 @@ -218,12 +190,9 @@ CA209168 100-2398 06ACHMD001 385-2116 -208-2226 160/MSANP/SG/UCP/PAR R01AI146123 RX 4265-057-JHU -09-2483 -GH-22-017 AID.2052-01011025CRT W81XWH-10-2-0090 90101534 @@ -238,14 +207,12 @@ IDSEP150041-01-00 54754C P3517 7802 U10DD000183 911-2431 -65398 R36HS026334 CNVA00061461 (134... 90106137 11-019 NAG511894 13-06738 -38762 80NSSC20K1069 227111 IS-0005 @@ -257,21 +224,7 @@ SFPRF8-13 966071 2014-02-06-JHU 02824-2 -278924 674-C-00-10-00128-00 -504406 -74536 -INV-009240 (1212721) -051077 -90105978 -BCS-1155233 -133096CSDG1900101 -R01NS052179 -231872 -90069353 -INV-006006 -R01CA092339 -ROI2016-912 ABNEU0169N AID-388-A-17-00002 Endoscopic assisted @@ -287,11 +240,9 @@ HHSN271200900013P R01AI066025 90102712 SHH4429g -850044 TTX-080 0835532N 7093553 -36204 U10CA180886 CHTF919GUS25 1729065 @@ -306,7 +257,6 @@ VX12-809-105 R01DK099188 259114Z99A 3U01AI066726-04S1 -39112 BYM338B2203E1 AKB-9778 RY001 061743001 @@ -318,17 +268,13 @@ R36D001162 A03-2546 CAPE-P11-292 90106024 -34248 -02903 373-2069 NNX07AC57G Task Order 71 CW88314-128072 R15B5200112 -100876 K08DA016160 INV-052361 -180345 H382085 R01HD038252 3U01AI035042-18S1 @@ -356,13 +302,11 @@ Task Order #77- Hatt 1527377 MRSG-07-284-01-CCE 638-2113 -34190 R01HL072488 MACS-2021-031602 N01-HD-2-3342 NNX13AG20G 3000031332 -72208 202232JHU R01EY014596 C3HR-02 @@ -371,12 +315,10 @@ HHSN266200400073C R01AI067371 R21HD087811 0725626U -67738 WO30085 90060695 R03DA035691 72061720CA00005 -114427 WO5051415 R-1806-02785 F32DK074294 @@ -386,8 +328,6 @@ NNX17AI54G P30NR011409 927117288 1433595 -7148704 -29773 0000008493 06P-009 G13LM010198 @@ -402,7 +342,6 @@ BRSIMPCTQ0600100 P01AG004953 453-2006 N00014-12-1-0402 -31217 HOWARD COUNTY PUBLIC 200-00537.09 201092880 @@ -410,8 +349,6 @@ HOWARD COUNTY PUBLIC ECU-MG-301 25284-06-324 R36MH084622 -722419 -39519 W81XWH0510407 H99076CT 2F-60288 @@ -429,11 +366,9 @@ R03AA013723 DMR1006805 F32HL144121 90ZJ0039 -37394 OFED0011-02 CPIP 23-100 221-2015 -72452 NNG04GJ20G TOK-200-05 41500616 @@ -447,8 +382,6 @@ RES507839 RES507840 HST-GO-15173.011-A U01CK000337 R01EY009769 -37830 -0504 R01AI118505 TUL-SCC-556459-18 19 1060810 @@ -464,8 +397,6 @@ N000140910588 INV-009067 (1212270) R21EB015555 INV-016877 -62451 -INV-001763 INV-006711 (1017055) 22006.03 K08NS069815 @@ -473,7 +404,6 @@ INV-008972 (1125119) 62250764-138575 PDY5807 INV-004357 -164872 STX10069009GR052 S06-VA10184 BCS-1528400 @@ -489,14 +419,12 @@ TUL-HSC-291-11 12 2011-02-001 110230 110009 542029 78001 -3305099 R01HG005840 R21AG034523 R01HL092259 VX09-809-102 20020637 N000140610160 -1736414 PG-266664-19 RFA04-040 W81XWH-15-1-0159 @@ -517,12 +445,10 @@ PCA - 2015 19 R01MH060626 CRA FC NL - JHU 2016 DE-SC0020428 -166170 SGA07-002 63030336-257650 18-0012-CA-AL LCI699C2301 -233238 # 36788 RC103133-JHU R01HD077882 @@ -541,7 +467,6 @@ CNS-1228443 20036262 MUSC18-071-8A840 HSTGO1014401A -5465 1908-06846 DSP-MD-01 170-2249 @@ -553,7 +478,6 @@ SP0077551 60063489 SOW #BO29337 TO JHU APR 2013-A1 CA2097FG -14607 205.478 A-8445 Project #0316-012 @@ -562,7 +486,6 @@ Project #0316-012 INV-008465 (1178578) 57720000000257 R01AI147681 -29767 16SDG27130006 Q202024 T32CA009243 @@ -573,7 +496,6 @@ SR00001603 2023-MSCRFF-6133 K08HL04396 DMS 0805531 -104366 1C06RR029946-01 NCHH05C0000 JHU IO#90052920 @@ -588,7 +510,6 @@ KG080137 3R01AI063764-05S1 STX10069010GR041 16118841 Hasan , Ran -36643 LT000125 2009-L-WU CHE-1360692 VA69D-P-0446 @@ -608,25 +529,21 @@ RX4265-072-JHU CFTY720DUS18T INV-051423 256-00 -544650 PUR01-0000028863 1R01AA017704-01A1 H982301010429 1802-05365 T15LM007452 0823481 -0513-005 2018-MSCRFF-4250 R21AI106491 S00486-01 W81XWH2010922 CH945 -1820784 1218.22 68-3A75-18-026 90031205 PHPA-S1659 -206-2395 PCYC-1111 AWD0001238 (133676-1 3R24DK064388-07S1 @@ -634,7 +551,6 @@ AWD0001238 (133676-1 FA251822C0010 NIHJHU102260 R01DA049301 -37156 HHSI236201600011C CAHMI2017.860.001 Wilson Pharma Fndtn @@ -642,23 +558,14 @@ CO# 39699 BMS747158-301 CTA PC040800 90032570 -22640 -157-405 -90080445 -90032366 CM287 R03MH089583 268350-5116197 -242757 -90063157 -90062149 EA000200642 1814919 UNIVERSITY OF EDINBU -128 NNX10AD79G W81XWH-07-1-0451 -32774 R21EY021897 JWST-GO-02130.003-A MDSS-2018-0339 @@ -677,12 +584,9 @@ SR00002000 AGS-1157463 110268-0719 01S 7487-CL-0209 -90081418 2014-MSCRFF-0665 N00244-20-1-0001 -201907199 ARC011 -126000 C0701A/203/ON/US NSP II 14-108 262164JHU @@ -691,10 +595,7 @@ NSP II 14-108 DMS0450083 A Support Cell-Speci TASK Order #78- Moni -1736448 1394-SUB R01AR057868 R01EB002849 -171237 -19IPLOI34770161 -90034463 \ No newline at end of file +19IPLOI34770161 \ No newline at end of file From 0564c46b9fa3433d71e4c028fbae56865e27e07b Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Fri, 6 Oct 2023 17:10:55 -0400 Subject: [PATCH 06/12] Add nanoTime BM --- .../eclipse/pass/client/nihms/NihmsPassClientServiceIT.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pass-nihms-loader/nihms-pass-client/src/test/java/org/eclipse/pass/client/nihms/NihmsPassClientServiceIT.java b/pass-nihms-loader/nihms-pass-client/src/test/java/org/eclipse/pass/client/nihms/NihmsPassClientServiceIT.java index 2b26c2f5..bde62510 100644 --- a/pass-nihms-loader/nihms-pass-client/src/test/java/org/eclipse/pass/client/nihms/NihmsPassClientServiceIT.java +++ b/pass-nihms-loader/nihms-pass-client/src/test/java/org/eclipse/pass/client/nihms/NihmsPassClientServiceIT.java @@ -195,6 +195,7 @@ public void shouldFindNihGrantAwardNumber() throws IOException { */ @Test public void shouldFindNonNormalizedNihGrantAwardNumber() throws IOException, URISyntaxException { + long startTime = System.nanoTime(); URI testAwardNumberUri = Objects.requireNonNull(NihmsPassClientServiceTest.class .getResource("/valid_award_numbers.csv")).toURI(); List awardNumbers = Files.readAllLines(Paths.get(testAwardNumberUri)); @@ -208,6 +209,9 @@ public void shouldFindNonNormalizedNihGrantAwardNumber() throws IOException, URI assertEquals(award, found.getAwardNumber()); assertEquals(grant.getId(), found.getId()); } + long endTime = System.nanoTime(); + double elapsedTimeInMS = (endTime - startTime) / 1_000_000.0; + System.out.println("Elapsed Time: " + elapsedTimeInMS + " ms"); } /** From 3143cee58cf5fcc60fd3119de2f24e6ad1d8329e Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Mon, 9 Oct 2023 15:29:22 -0400 Subject: [PATCH 07/12] Add sorting to grants --- .../pass/client/nihms/NihmsPassClientService.java | 15 ++++++++++----- .../client/nihms/NihmsPassClientServiceTest.java | 2 ++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/pass-nihms-loader/nihms-pass-client/src/main/java/org/eclipse/pass/client/nihms/NihmsPassClientService.java b/pass-nihms-loader/nihms-pass-client/src/main/java/org/eclipse/pass/client/nihms/NihmsPassClientService.java index 639de45a..53ce4266 100644 --- a/pass-nihms-loader/nihms-pass-client/src/main/java/org/eclipse/pass/client/nihms/NihmsPassClientService.java +++ b/pass-nihms-loader/nihms-pass-client/src/main/java/org/eclipse/pass/client/nihms/NihmsPassClientService.java @@ -16,8 +16,6 @@ package org.eclipse.pass.client.nihms; import java.io.IOException; -import java.util.Collections; -import java.util.Comparator; import java.util.List; import java.util.stream.Stream; @@ -184,18 +182,25 @@ public Grant findMostRecentGrantByAwardNumber(String awardNumber) throws IOExcep PassClientSelector grantSelector = new PassClientSelector(Grant.class); String normalizedAwardNumberFilter = ModelUtil.createAwardNumberQuery(awardNumber, AWARD_NUMBER_FLD); grantSelector.setFilter(normalizedAwardNumberFilter); + grantSelector.setSorting("-startDate"); Stream grantStream = passClient.streamObjects(grantSelector); grants = grantStream.toList(); - if (grants.size() == 1) { + if (grants.size() > 0) { + return grants.get(0); + } + + return null; + + /*if (grants.size() == 1) { return grants.get(0); } else if (grants.size() > 0) { Grant mostRecentGrant = Collections.max(grants, Comparator.comparing(Grant::getStartDate)); grantCache.put(awardNumber, mostRecentGrant.getId()); return mostRecentGrant; - } + }*/ - return null; + //return null; } /** diff --git a/pass-nihms-loader/nihms-pass-client/src/test/java/org/eclipse/pass/client/nihms/NihmsPassClientServiceTest.java b/pass-nihms-loader/nihms-pass-client/src/test/java/org/eclipse/pass/client/nihms/NihmsPassClientServiceTest.java index 2591923b..d0804252 100644 --- a/pass-nihms-loader/nihms-pass-client/src/test/java/org/eclipse/pass/client/nihms/NihmsPassClientServiceTest.java +++ b/pass-nihms-loader/nihms-pass-client/src/test/java/org/eclipse/pass/client/nihms/NihmsPassClientServiceTest.java @@ -53,6 +53,7 @@ import org.eclipse.pass.support.client.model.Submission; import org.eclipse.pass.support.client.model.User; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; @@ -65,6 +66,7 @@ * @author Karen Hanson */ @ExtendWith(MockitoExtension.class) +@Disabled("While testing performance") public class NihmsPassClientServiceTest { private static final String grant1Id = "1"; From 255def2702daf457d021dd43e98aeb2accd463b4 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Mon, 9 Oct 2023 15:59:59 -0400 Subject: [PATCH 08/12] Refactor to using stream to find max --- .../pass/client/nihms/NihmsPassClientService.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pass-nihms-loader/nihms-pass-client/src/main/java/org/eclipse/pass/client/nihms/NihmsPassClientService.java b/pass-nihms-loader/nihms-pass-client/src/main/java/org/eclipse/pass/client/nihms/NihmsPassClientService.java index 53ce4266..a510abbb 100644 --- a/pass-nihms-loader/nihms-pass-client/src/main/java/org/eclipse/pass/client/nihms/NihmsPassClientService.java +++ b/pass-nihms-loader/nihms-pass-client/src/main/java/org/eclipse/pass/client/nihms/NihmsPassClientService.java @@ -16,7 +16,9 @@ package org.eclipse.pass.client.nihms; import java.io.IOException; +import java.util.Comparator; import java.util.List; +import java.util.Optional; import java.util.stream.Stream; import org.apache.commons.collections4.CollectionUtils; @@ -184,11 +186,19 @@ public Grant findMostRecentGrantByAwardNumber(String awardNumber) throws IOExcep grantSelector.setFilter(normalizedAwardNumberFilter); grantSelector.setSorting("-startDate"); Stream grantStream = passClient.streamObjects(grantSelector); - grants = grantStream.toList(); + + Optional mostRecentGrant = grantStream.max(Comparator.comparing(Grant::getStartDate)); + + if (mostRecentGrant.isPresent()) { + grantCache.put(awardNumber, mostRecentGrant.get().getId()); + return mostRecentGrant.get(); + } + + /*grants = grantStream.toList(); if (grants.size() > 0) { return grants.get(0); - } + }*/ return null; From 7060a9056522d1790b538b8d9d329bf5a892befb Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Mon, 9 Oct 2023 16:10:50 -0400 Subject: [PATCH 09/12] Revert back to original algo --- .../client/nihms/NihmsPassClientService.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/pass-nihms-loader/nihms-pass-client/src/main/java/org/eclipse/pass/client/nihms/NihmsPassClientService.java b/pass-nihms-loader/nihms-pass-client/src/main/java/org/eclipse/pass/client/nihms/NihmsPassClientService.java index a510abbb..8fec7c79 100644 --- a/pass-nihms-loader/nihms-pass-client/src/main/java/org/eclipse/pass/client/nihms/NihmsPassClientService.java +++ b/pass-nihms-loader/nihms-pass-client/src/main/java/org/eclipse/pass/client/nihms/NihmsPassClientService.java @@ -16,6 +16,7 @@ package org.eclipse.pass.client.nihms; import java.io.IOException; +import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Optional; @@ -184,33 +185,33 @@ public Grant findMostRecentGrantByAwardNumber(String awardNumber) throws IOExcep PassClientSelector grantSelector = new PassClientSelector(Grant.class); String normalizedAwardNumberFilter = ModelUtil.createAwardNumberQuery(awardNumber, AWARD_NUMBER_FLD); grantSelector.setFilter(normalizedAwardNumberFilter); - grantSelector.setSorting("-startDate"); + //grantSelector.setSorting("-startDate"); Stream grantStream = passClient.streamObjects(grantSelector); - Optional mostRecentGrant = grantStream.max(Comparator.comparing(Grant::getStartDate)); + /*Optional mostRecentGrant = grantStream.max(Comparator.comparing(Grant::getStartDate)); if (mostRecentGrant.isPresent()) { grantCache.put(awardNumber, mostRecentGrant.get().getId()); return mostRecentGrant.get(); - } + }*/ - /*grants = grantStream.toList(); + grants = grantStream.toList(); - if (grants.size() > 0) { + /*if (grants.size() > 0) { return grants.get(0); }*/ - return null; + //return null; - /*if (grants.size() == 1) { + if (grants.size() == 1) { return grants.get(0); } else if (grants.size() > 0) { Grant mostRecentGrant = Collections.max(grants, Comparator.comparing(Grant::getStartDate)); grantCache.put(awardNumber, mostRecentGrant.getId()); return mostRecentGrant; - }*/ + } - //return null; + return null; } /** From f87f99626dbd61611f1e3817a052572bcb2f2c06 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Mon, 9 Oct 2023 17:39:47 -0400 Subject: [PATCH 10/12] Remove award numbers -Revert back to original -Cleanup --- .../client/nihms/NihmsPassClientService.java | 22 +- .../nihms/NihmsPassClientServiceTest.java | 2 - .../test/resources/valid_award_numbers.csv | 379 ++---------------- 3 files changed, 35 insertions(+), 368 deletions(-) diff --git a/pass-nihms-loader/nihms-pass-client/src/main/java/org/eclipse/pass/client/nihms/NihmsPassClientService.java b/pass-nihms-loader/nihms-pass-client/src/main/java/org/eclipse/pass/client/nihms/NihmsPassClientService.java index 8fec7c79..5feaaff1 100644 --- a/pass-nihms-loader/nihms-pass-client/src/main/java/org/eclipse/pass/client/nihms/NihmsPassClientService.java +++ b/pass-nihms-loader/nihms-pass-client/src/main/java/org/eclipse/pass/client/nihms/NihmsPassClientService.java @@ -19,7 +19,6 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; -import java.util.Optional; import java.util.stream.Stream; import org.apache.commons.collections4.CollectionUtils; @@ -185,24 +184,9 @@ public Grant findMostRecentGrantByAwardNumber(String awardNumber) throws IOExcep PassClientSelector grantSelector = new PassClientSelector(Grant.class); String normalizedAwardNumberFilter = ModelUtil.createAwardNumberQuery(awardNumber, AWARD_NUMBER_FLD); grantSelector.setFilter(normalizedAwardNumberFilter); - //grantSelector.setSorting("-startDate"); Stream grantStream = passClient.streamObjects(grantSelector); - - /*Optional mostRecentGrant = grantStream.max(Comparator.comparing(Grant::getStartDate)); - - if (mostRecentGrant.isPresent()) { - grantCache.put(awardNumber, mostRecentGrant.get().getId()); - return mostRecentGrant.get(); - }*/ - grants = grantStream.toList(); - /*if (grants.size() > 0) { - return grants.get(0); - }*/ - - //return null; - if (grants.size() == 1) { return grants.get(0); } else if (grants.size() > 0) { @@ -343,9 +327,7 @@ public List findSubmissionsByPublicationAndUserId(String pubId, Stri subSelector.setFilter(subFilter); subSelector.setInclude("publication", "repositories", "submitter", "grants"); PassClientResult subResult = passClient.selectObjects(subSelector); - List submissions = subResult.getObjects(); - - return submissions; + return subResult.getObjects(); } /** @@ -641,7 +623,7 @@ public boolean updateDeposit(Deposit deposit) throws IOException { } private static String userIdPubIdKey(String userId, String pubId) { - return userId.toString() + pubId.toString(); + return userId.toString() + pubId; } } diff --git a/pass-nihms-loader/nihms-pass-client/src/test/java/org/eclipse/pass/client/nihms/NihmsPassClientServiceTest.java b/pass-nihms-loader/nihms-pass-client/src/test/java/org/eclipse/pass/client/nihms/NihmsPassClientServiceTest.java index d0804252..2591923b 100644 --- a/pass-nihms-loader/nihms-pass-client/src/test/java/org/eclipse/pass/client/nihms/NihmsPassClientServiceTest.java +++ b/pass-nihms-loader/nihms-pass-client/src/test/java/org/eclipse/pass/client/nihms/NihmsPassClientServiceTest.java @@ -53,7 +53,6 @@ import org.eclipse.pass.support.client.model.Submission; import org.eclipse.pass.support.client.model.User; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; @@ -66,7 +65,6 @@ * @author Karen Hanson */ @ExtendWith(MockitoExtension.class) -@Disabled("While testing performance") public class NihmsPassClientServiceTest { private static final String grant1Id = "1"; diff --git a/pass-nihms-loader/nihms-pass-client/src/test/resources/valid_award_numbers.csv b/pass-nihms-loader/nihms-pass-client/src/test/resources/valid_award_numbers.csv index f054a5c0..d85f7d37 100644 --- a/pass-nihms-loader/nihms-pass-client/src/test/resources/valid_award_numbers.csv +++ b/pass-nihms-loader/nihms-pass-client/src/test/resources/valid_award_numbers.csv @@ -12,142 +12,46 @@ K24HD052559 918-2088 VUMC43851 NNX13AK97G -476-2127 22-A0-00-1008755 -122-1 TA-12-049 FA9550-23-1-0531 -57005932 IPA-4 R21NS054146 HST-GO-14211.002-A N00014-19-1-2673 UNF-18-1023 -90106091 BR4380 N00014-16-1-2406 R01AI24576 19PRE34380815 -U18TR003780 ATM0101064 JHU IO#90056635 -0005127 -S-901-3K5-006 108659-54-HAMM U58DP002653 JHU Grant #100710 -A1443-014 303000309 1-05-RA-50 -90080076 R21OH009829 -0254-5953-4609 CV6185 -MUSC12-057 Biocon Bressler ISA R01DK57998 MB200221702 OF-50 NNG06GF23G -19-028-ASP -A Peer-Based Benchma TV5600-CNS-20007 SOW#BN40423 13042486 Winner , La -90056708 -FY16.610.001 W81XWH-17-1-0630 BresslerGenentechISA T32AG000120 -90072100 -W81XWH-14-1-0396 -90092667 NA 00038389 RFA-MD-11-1 -801789 DEC-46 -200168785 4391CPFF0702 R43HG008709 Project #0117-003 (J -90071038 -100028501 -679-2232 N00244-13-1-0001 G13LM07528 -5007386-SERV F31NR020760 -UF09132 -R21GM146105 -INV-007197 (1063942) -JHU IO#90050185 -N000140910867 -INV-060110 -90088694 -U01CK000554 -NAG511034 -Sub # 1500181 -20120865 -R01GM079239 -CC5013-MDS-002 -FY18-CO#38391 -610706 -NNX11AK65G -OPP1046227 -10-2438-015 -R01EY017205 -subaward 203-8371 -NA 00043632 -W911NF-10-1-0326 -HST-GO-15840.034-A -INV-035001 -HST-GO-12603.01-A -HR001107P0019 -90WT000501 -SES-1628179 -4-312-0212746-50719L -43297444 -ANG206 -NNX09AC98A -492C00030002400CRUS0 -f77988f3-bb58-4848-9 -G-1904-00069 -T03MC07645 -90074741 -MUSC09-093 -2018-7642 -R01DA004133 -U01HL096812 -304-2279 -21-06093 -A03-2466 -K99AI151094 -90085073 -2014-MSCRFI-0715 -R01NS117604 -R21HD060169 -R36AG083325 -CP 12663 2 23 A25 -0630223N -U1QHP28710 -12-049 -PO # 20224722 -S07RR18230 -VX09-222-103 -IO# 90041542 -DHMH-OPASS-17-17108 -80NSSC19K0589 -PGP0007 2012 -HDTRA11110063 -5106332 -3000037037 -ZEN062 -VAWA-2010-1641 -UCHC6-55818869 -R01NS048206 -0314-012 12 (KANG) -IM101-226-111 -ADU-CL-03 -NAG512518 12-2438-01E INV-043944 AID-391-A-14-00002 @@ -184,68 +88,34 @@ U19CA113341 K08NS062890 DRL1019306 202617231-1 -633-2123 HHSN271200700264P CA209168 -100-2398 -06ACHMD001 -385-2116 160/MSANP/SG/UCP/PAR R01AI146123 RX 4265-057-JHU AID.2052-01011025CRT W81XWH-10-2-0090 -90101534 -R01MH129397 VERTEX010-081808 -90102180 AID-279-LA-14-00006 NNX17AC42G P022A060041 -5 IDSEP150041-01-00 54754C P3517 7802 U10DD000183 -911-2431 -R36HS026334 CNVA00061461 (134... -90106137 -11-019 -NAG511894 -13-06738 -80NSSC20K1069 -227111 -IS-0005 CON-80002993(GR11280 -SPRAY-CB 80NSSC19K0856 -S-00057 -SFPRF8-13 -966071 -2014-02-06-JHU -02824-2 674-C-00-10-00128-00 ABNEU0169N AID-388-A-17-00002 Endoscopic assisted R01AI107028 -213-2161 -1701168 548 -R01AA10158 FFP-A-11-00006-02 D4320C00015 -080-01000-R50004 HHSN271200900013P -2013-MSCRFF-051 -R01AI066025 90102712 -SHH4429g -TTX-080 -0835532N -7093553 U10CA180886 CHTF919GUS25 -1729065 R01AI052468 R01MH103325 2019-MSCRFF-5124 @@ -254,8 +124,6 @@ BDS-SAMLAB EP08011011 2022-0414 VX12-809-105 -R01DK099188 -259114Z99A 3U01AI066726-04S1 BYM338B2203E1 AKB-9778 RY001 @@ -265,23 +133,6 @@ R21DK073755 H98230-16D0026/0006 SES0421715 R36D001162 -A03-2546 -CAPE-P11-292 -90106024 -373-2069 -NNX07AC57G -Task Order 71 -CW88314-128072 -R15B5200112 -K08DA016160 -INV-052361 -H382085 -R01HD038252 -3U01AI035042-18S1 -2013-39320 -R01GM105956 -R03EY014416 -361040-08110-7431-02 R01HL085567 16-JHU-01 24XS126 @@ -385,217 +236,53 @@ R01EY009769 R01AI118505 TUL-SCC-556459-18 19 1060810 -160502-0902 -0000205 -CW2267907 -R21HG005745 -0515328U -B832023 -JWST-ERS-01366.064-A -HST-GO-14268.009-A -N000140910588 -INV-009067 (1212270) -R21EB015555 -INV-016877 INV-006711 (1017055) -22006.03 -K08NS069815 INV-008972 (1125119) +INV-008802 (1211544) +INV-008465 (1178578) +AWD0001238 (133676-1 +JWST-ERS-01366.064-A +HST-GO-14268.009-A 62250764-138575 -PDY5807 -INV-004357 STX10069009GR052 -S06-VA10184 -BCS-1528400 -0725357U -G-2020-13933 -IDE G010230 -870-2218 -BCTR0402523 -INV-008802 (1211544) -NNX11AG50G -Wiki 2010 -TUL-HSC-291-11 12 -2011-02-001 +STX10069010GR041 +3R01AI063764-05S1 +3R24DK064388-07S1 +TASK Order #78- Moni 110230 110009 542029 78001 -R01HG005840 +SP0077551 60063489 +110268-0719 01S +TUL-HSC-291-11 12 +TO JHU APR 2013-A1 +16118841 Hasan , Ran +LT000125 2009-L-WU +H76HA28436 +R01HD077882 +R01AR057868 +R01EB002849 +R01MH060626 +R21HG005745 +R21EB015555 R21AG034523 +R01HG005840 R01HL092259 -VX09-809-102 -20020637 -N000140610160 -PG-266664-19 -RFA04-040 -W81XWH-15-1-0159 -57008411 R01HS026599 -FY20.944.001 -FFCPSS -VUMC 85007 R01NR017433 -P0132842 R21CA249362 -C-02-60 -BMN-111-901 -1700034577 -H98230-15-1-0302 -80NSSC22K1246 -PCA - 2015 19 -R01MH060626 -CRA FC NL - JHU 2016 -DE-SC0020428 -SGA07-002 -63030336-257650 -18-0012-CA-AL -LCI699C2301 -# 36788 -RC103133-JHU -R01HD077882 -G-71602 -M333001 HS9472 -DL6049-0417 -N01-CM-17104 -NNFF-Hansen -B692063 -F32DK076257 -1411-S001 -ATM0551127 -C532053 -190.1D -CNS-1228443 -20036262 -MUSC18-071-8A840 -HSTGO1014401A -1908-06846 -DSP-MD-01 -170-2249 -5267-01 -80NSSC22K1698 +R21AI106491 +R03MH089583 5(GG09619-04) -SP0077551 60063489 -918-2098 -SOW #BO29337 -TO JHU APR 2013-A1 -CA2097FG -205.478 -A-8445 -Project #0316-012 -200-2007-23689 -0924815 -INV-008465 (1178578) -57720000000257 -R01AI147681 -16SDG27130006 -Q202024 -T32CA009243 -ODAP0701264 -R01NS076357 -CDFV89G12101 -SR00001603 -2023-MSCRFF-6133 -K08HL04396 -DMS 0805531 1C06RR029946-01 -NCHH05C0000 -JHU IO#90052920 -ICT0011-A-7 -80NSSC23K1142 -634-2016 -2105 G TA137 -RFA-MD-09-2 -Project Agreement 2 -KG080137 -3F-32761 -3R01AI063764-05S1 -STX10069010GR041 -16118841 Hasan , Ran -LT000125 2009-L-WU -CHE-1360692 -VA69D-P-0446 -R21DA020777 -H76HA28436 -200726401 -150998-005 Allerga -U24CA271079 -0000002876 -NEDP104232 -HR0011-16-C-0139 -DESC00004857 -The Clinical Impact -280-2076 -4216SC -RX4265-072-JHU -CFTY720DUS18T -INV-051423 -256-00 -PUR01-0000028863 1R01AA017704-01A1 -H982301010429 -1802-05365 -T15LM007452 -0823481 -2018-MSCRFF-4250 -R21AI106491 -S00486-01 -W81XWH2010922 -CH945 -1218.22 -68-3A75-18-026 -90031205 -PHPA-S1659 -PCYC-1111 -AWD0001238 (133676-1 -3R24DK064388-07S1 -90073586 -FA251822C0010 -NIHJHU102260 -R01DA049301 -HHSI236201600011C -CAHMI2017.860.001 -Wilson Pharma Fndtn -CO# 39699 -BMS747158-301 CTA -PC040800 -90032570 -CM287 -R03MH089583 -268350-5116197 -EA000200642 -1814919 -UNIVERSITY OF EDINBU -NNX10AD79G -W81XWH-07-1-0451 R21EY021897 JWST-GO-02130.003-A -MDSS-2018-0339 -GAGA-2010-C-0141 -W911NF2120211 -4500003555 +VUMC 85007 D71TW008193 -154593 -NNX10AM65G -804-2014 -INV-044571 -VUMC35628 -217-2881 -HHSI245201501072P -SR00002000 -AGS-1157463 -110268-0719 01S -7487-CL-0209 -2014-MSCRFF-0665 -N00244-20-1-0001 -ARC011 -C0701A/203/ON/US -NSP II 14-108 -262164JHU -0875GPC538 -6483-15-AM-02 -DMS0450083 -A Support Cell-Speci -TASK Order #78- Moni -1394-SUB -R01AR057868 -R01EB002849 -19IPLOI34770161 \ No newline at end of file +HR0011-16-C-0139 +F32DK076257 +R01AI147681 +1R01AA017704-01A1 +R01NS076357 +W81XWH-15-1-0159 +N000140910588 \ No newline at end of file From 4baf87f9675d2d863df7ad3e1d0e4cd220aa7c37 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Mon, 9 Oct 2023 17:44:34 -0400 Subject: [PATCH 11/12] Remove nanoTime --- .../eclipse/pass/client/nihms/NihmsPassClientServiceIT.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pass-nihms-loader/nihms-pass-client/src/test/java/org/eclipse/pass/client/nihms/NihmsPassClientServiceIT.java b/pass-nihms-loader/nihms-pass-client/src/test/java/org/eclipse/pass/client/nihms/NihmsPassClientServiceIT.java index bde62510..2b26c2f5 100644 --- a/pass-nihms-loader/nihms-pass-client/src/test/java/org/eclipse/pass/client/nihms/NihmsPassClientServiceIT.java +++ b/pass-nihms-loader/nihms-pass-client/src/test/java/org/eclipse/pass/client/nihms/NihmsPassClientServiceIT.java @@ -195,7 +195,6 @@ public void shouldFindNihGrantAwardNumber() throws IOException { */ @Test public void shouldFindNonNormalizedNihGrantAwardNumber() throws IOException, URISyntaxException { - long startTime = System.nanoTime(); URI testAwardNumberUri = Objects.requireNonNull(NihmsPassClientServiceTest.class .getResource("/valid_award_numbers.csv")).toURI(); List awardNumbers = Files.readAllLines(Paths.get(testAwardNumberUri)); @@ -209,9 +208,6 @@ public void shouldFindNonNormalizedNihGrantAwardNumber() throws IOException, URI assertEquals(award, found.getAwardNumber()); assertEquals(grant.getId(), found.getId()); } - long endTime = System.nanoTime(); - double elapsedTimeInMS = (endTime - startTime) / 1_000_000.0; - System.out.println("Elapsed Time: " + elapsedTimeInMS + " ms"); } /** From a9421dbf2f8bac61b98eb68ed0b73fd1ac77eb3a Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Tue, 10 Oct 2023 09:30:35 -0400 Subject: [PATCH 12/12] Remove redundant test cases --- .../eclipse/pass/loader/nihms/TransformAndLoadSmokeIT.java | 6 +++--- .../src/test/resources/data/inprocess_NihmsData.csv | 2 -- .../src/test/resources/data/noncompliant_NihmsData.csv | 5 +---- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/pass-nihms-loader/nihms-data-transform-load/src/test/java/org/eclipse/pass/loader/nihms/TransformAndLoadSmokeIT.java b/pass-nihms-loader/nihms-data-transform-load/src/test/java/org/eclipse/pass/loader/nihms/TransformAndLoadSmokeIT.java index 93db823a..9c371e5c 100644 --- a/pass-nihms-loader/nihms-data-transform-load/src/test/java/org/eclipse/pass/loader/nihms/TransformAndLoadSmokeIT.java +++ b/pass-nihms-loader/nihms-data-transform-load/src/test/java/org/eclipse/pass/loader/nihms/TransformAndLoadSmokeIT.java @@ -65,15 +65,15 @@ public void smokeTestLoadAndTransform(WireMockRuntimeInfo wmRuntimeInfo) throws //make sure RepositoryCopies are all in before moving on so we can be sure the counts are done. repoCopySelector.setFilter(RSQL.notEquals("id", "-1")); List repositoryCopies = passClient.selectObjects(repoCopySelector).getObjects(); - assertEquals(26, repositoryCopies.size()); + assertEquals(23, repositoryCopies.size()); publicationSelector.setFilter(RSQL.notEquals("id", "-1")); List publications = passClient.selectObjects(publicationSelector).getObjects(); - assertEquals(37, publications.size()); + assertEquals(32, publications.size()); submissionSelector.setFilter(RSQL.notEquals("id", "-1")); List submissions = passClient.selectObjects(submissionSelector).getObjects(); - assertEquals(37, submissions.size()); + assertEquals(32, submissions.size()); //reset file names: File downloadDir = new File(path); diff --git a/pass-nihms-loader/nihms-data-transform-load/src/test/resources/data/inprocess_NihmsData.csv b/pass-nihms-loader/nihms-data-transform-load/src/test/resources/data/inprocess_NihmsData.csv index bb5561dc..1f81e00f 100644 --- a/pass-nihms-loader/nihms-data-transform-load/src/test/resources/data/inprocess_NihmsData.csv +++ b/pass-nihms-loader/nihms-data-transform-load/src/test/resources/data/inprocess_NihmsData.csv @@ -10,11 +10,9 @@ PMID,PMCID,NIHMSID,Grant Number,PI Name,Publication Date,NIHMS file deposited,NI 78787878,,787878,P20 HHHHHH,"COFFEE, K",6/13/2017,5/30/2017,5/30/2017,7/10/2017,,Article O,not used,not used,not used,no,not used 89898989,,898989,N01 IIIIII,"BLUEMKE, DAVID A",11/7/2017,6/16/2017,6/26/2017,8/9/2017,,Article P,not used,not used,not used,no,not used 89898989,,898989,UL1 JJJJJJ,"AQUA, E",11/7/2017,6/16/2017,6/26/2017,8/9/2017,,Article P,not used,not used,not used,no,not used -90909090,,909090,P30 KKKKKK,"CLAY, F",1/1/2018,2/1/2018,2/1/2018,,,Article Q,not used,not used,not used,no,not used 98989898,,989898,U01 LLLLLL,"MINT, A",7/24/2017,1/26/2018,1/26/2018,,,Article R,not used,not used,not used,no,not used 87878787,,878787,T32 MMMMMM,"GRAY, I",8/1/2017,8/13/2017,,,,Article S,not used,not used,not used,no,not used 87878787,,878787,T32 NNNNNN,"IVORY, H",8/1/2017,8/13/2017,,,,Article S,not used,not used,not used,no,not used 76767676,,767676,R01 OOOOOO,"MAROON, P",12/1/2017,12/20/2017,12/20/2017,1/9/2018,,Article T,not used,not used,not used,no,not used 65656565,,656565,R01 PPPPPP,"SAGE, X",9/22/2017,9/18/2017,,10/2/2017,,Article U,not used,not used,not used,no,not used 65656565,,656565,R01 QQQQQQ,"SAGE, X",9/22/2017,9/18/2017,,10/2/2017,,Article U,not used,not used,not used,no,not used -54545454,,545454,T32 RRRRRR,"SAGE, X",9/22/2017,9/18/2017,,10/2/2017,,Article U,not used,not used,not used,no,not used diff --git a/pass-nihms-loader/nihms-data-transform-load/src/test/resources/data/noncompliant_NihmsData.csv b/pass-nihms-loader/nihms-data-transform-load/src/test/resources/data/noncompliant_NihmsData.csv index 4ea40371..e4df1255 100644 --- a/pass-nihms-loader/nihms-data-transform-load/src/test/resources/data/noncompliant_NihmsData.csv +++ b/pass-nihms-loader/nihms-data-transform-load/src/test/resources/data/noncompliant_NihmsData.csv @@ -1,17 +1,14 @@ PMID,PMCID,NIHMSID,Grant Number,PI Name,Publication Date,NIHMS file deposited,NIHMS initial approval,NIHMS tagging complete,NIHMS final approval,Article Title,First Author Name,Journal Title,Journal Publisher,Method A Journal,NIHMS Person 13131313,,131313,T32 KKKKKK,"CHOCOLATE, A",7/1/2017,2/23/2016,3/9/2016,3/21/2016,,Article PQ,not used,not used,not used,no,not used 24242424,,,T32 LLLLLL,"CRIMSON, B",6/1/2017,,,,,Article OP,not used,not used,not used,no,not used -35353535,,,K23 MMMMMM,"NAVY, E",10/1/2017,,,,,Article NO,not used,not used,not used,no,not used 46464646,,,T32 NNNNNN,"IVORY, H",2/1/2018,,,,,Article MN,not used,not used,not used,no,not used 57575757,,,TL1 OOOOOO,"MINT, D",12/1/2017,,,,,Article LM,not used,not used,not used,no,not used -68686868,,686868,T32 PPPPPP,"RED, F",3/1/2017,8/15/2017,,,,Article KL,not used,not used,not used,no,not used 79797979,,,R01 QQQQQQ,"SAGE, X",4/1/2017,,,,,Article JK,not used,not used,not used,no,not used 79797979,,,R01 RRRRRR,"TEAL, K",3/1/2017,,,,,Article V,not used,not used,not used,no,not used 79797979,,,K24 SSSSSS,"TEAL, K",3/1/2017,,,,,Article V,not used,not used,not used,no,not used 80808080,,,N01 TTTTTT,"SLATE, M",4/1/2017,,,,,Article EF,not used,not used,not used,no,not used 97979797,,,P50 UUUUUU,"SANDY, F",4/15/2017,,,,,Article FG,not used,not used,not used,no,not used -86868686,,,P30 VVVVVV,"TAN, R",3/1/2017,,,,,Article GH,not used,not used,not used,no,not used -75757575,,,R01 WWWWWW,"GOLDEN, Q",2/1/2017,,,,,Article HI,not used,not used,not used,no,not used +86868686,,,P30 VVVVVV,"TAN, R",3/1/2017,,,,,Article GH,not used,not used,not used,no,not usedd 64646464,,646464,T32 XXXXXX,"LIME, N",5/1/2017,9/15/2017,,,,Article IJ,not used,not used,not used,no,not used 53535353,,,R01 YYYYYY,"CYAN, S",12/1/2017,,,,,Article AB,not used,not used,not used,no,not used 42424242,,424242,P30 ZZZZZZ,"OLIVE, K",5/1/2017,1/30/2017,2/9/2017,3/7/2017,,Article CD,not used,not used,not used,no,not used