Merge pull request #17 from bento-platform/qa/v3.1

v3.1.2

LICENSE

@@ -0,0 +1,165 @@
+                   GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.

Makefile

@@ -114,7 +114,7 @@ stop-%:
 
 
 # Clean up
-clean-all: clean-api clean-api-alpine clean-gateway clean-drs
+clean-all: clean-api clean-gateway clean-drs
 
 clean-gateway:
 	docker rm ${GOHAN_GATEWAY_CONTAINER_NAME} --force; \

etc/example.env

@@ -47,6 +47,7 @@ GOHAN_API_CONTAINERIZED_VCF_PATH=/app/vcfs
 GOHAN_API_GTF_PATH=/path/to/gtfs/on/host/machine
 GOHAN_API_CONTAINERIZED_GTF_PATH=/app/gtfs
 
+GOHAN_API_BULK_INDEXING_CAP=10000
 GOHAN_API_FILE_PROC_CONC_LVL=3
 GOHAN_API_LINE_PROC_CONC_LVL=1000
 

src/api/main.go

@@ -35,6 +35,7 @@ func main() {
 
 		"\tVCF Directory Path : %s \n"+
 		"\tGTF Directory Path : %s \n"+
+		"\tBulk Indexing Cap : %d\n"+
 		"\tFile Processing Concurrency Level : %d\n"+
 		"\tLine Processing Concurrency Level : %d\n"+
 		"\tElasticsearch Url : %s \n"+
@@ -55,6 +56,7 @@ func main() {
 		cfg.Debug,
 		cfg.Api.VcfPath,
 		cfg.Api.GtfPath,
+		cfg.Api.BulkIndexingCap,
 		cfg.Api.FileProcessingConcurrencyLevel,
 		cfg.Api.LineProcessingConcurrencyLevel,
 		cfg.Elasticsearch.Url, cfg.Elasticsearch.Username,
@@ -169,6 +171,7 @@ func main() {
 		// middleware
 		gam.MandateAssemblyIdAttribute)
 	e.GET("/variants/ingestion/requests", mvc.GetAllVariantIngestionRequests)
+	e.GET("/variants/ingestion/stats", mvc.VariantsIngestionStats)
 
 	// -- Genes
 	e.GET("/genes/overview", mvc.GetGenesOverview)
@@ -177,6 +180,7 @@ func main() {
 		gam.ValidateOptionalChromosomeAttribute)
 	e.GET("/genes/ingestion/requests", mvc.GetAllGeneIngestionRequests)
 	e.GET("/genes/ingestion/run", mvc.GenesIngest)
+	e.GET("/genes/ingestion/stats", mvc.VariantsIngestionStats)
 
 	// -- Workflows
 	e.GET("/workflows", mvc.WorkflowsGet)

src/api/models/config.go

@@ -8,6 +8,7 @@ type Config struct {
 		Port                           string `yaml:"port" envconfig:"GOHAN_API_INTERNAL_PORT"`
 		VcfPath                        string `yaml:"vcfPath" envconfig:"GOHAN_API_VCF_PATH"`
 		LocalVcfPath                   string `yaml:"localVcfPath" envconfig:"GOHAN_API_VCF_PATH"`
+		BulkIndexingCap                int    `yaml:"BulkIndexingCap" envconfig:"GOHAN_API_BULK_INDEXING_CAP"`
 		FileProcessingConcurrencyLevel int    `yaml:"fileProcessingConcurrencyLevel" envconfig:"GOHAN_API_FILE_PROC_CONC_LVL"`
 		LineProcessingConcurrencyLevel int    `yaml:"lineProcessingConcurrencyLevel" envconfig:"GOHAN_API_LINE_PROC_CONC_LVL"`
 		GtfPath                        string `yaml:"gtfPath" envconfig:"GOHAN_API_GTF_PATH"`

src/api/models/dtos.go

@@ -17,6 +17,8 @@ type BentoV2CompatibleVariantResponseDataModel struct {
 	GenotypeType string `json:"genotype_type"`
 }
 
+// --
+
 type VariantsResponseDTO struct {
 	Status  int                        `json:"status"`
 	Message string                     `json:"message"`
@@ -29,6 +31,8 @@ type VariantResponseDataModel struct {
 	Results   interface{} `json:"results"` // i.e.: []Variant or []string
 }
 
+// -- --
+
 type GenesResponseDTO struct {
 	Status  int    `json:"status"`
 	Message string `json:"message"`

src/api/mvc/genes.go

@@ -26,6 +26,13 @@ import (
 	"github.com/mitchellh/mapstructure"
 )
 
+func GenesIngestionStats(c echo.Context) error {
+	fmt.Printf("[%s] - GenesIngestionStats hit!\n", time.Now())
+	ingestionService := c.(*contexts.GohanContext).IngestionService
+
+	return c.JSON(http.StatusOK, ingestionService.GeneIngestionBulkIndexer.Stats())
+}
+
 func GenesIngest(c echo.Context) error {
 	fmt.Printf("[%s] - GenesIngest hit!\n", time.Now())
 	// trigger global ingestion background process

src/api/mvc/variants.go

@@ -32,6 +32,13 @@ import (
 	"github.com/mitchellh/mapstructure"
 )
 
+func VariantsIngestionStats(c echo.Context) error {
+	fmt.Printf("[%s] - VariantsIngestionStats hit!\n", time.Now())
+	ingestionService := c.(*contexts.GohanContext).IngestionService
+
+	return c.JSON(http.StatusOK, ingestionService.IngestionBulkIndexer.Stats())
+}
+
 func VariantsGetByVariantId(c echo.Context) error {
 	fmt.Printf("[%s] - VariantsGetByVariantId hit!\n", time.Now())
 	// retrieve variant Ids from query parameter (comma separated)
@@ -226,11 +233,13 @@ func VariantsIngest(c echo.Context) error {
 				// free up a spot in the queue
 				defer func() { <-ingestionService.ConcurrentFileIngestionQueue }()
 
+				fmt.Printf("Begin running %s !\n", gzippedFileName)
 				reqStat.State = ingest.Running
 				ingestionService.IngestRequestChan <- reqStat
 
 				// ---	 decompress vcf.gz
 
+				fmt.Printf("Decompressing %s !\n", gzippedFileName)
 				gzippedFilePath := fmt.Sprintf("%s%s", vcfPath, gzippedFileName)
 				r, err := os.Open(gzippedFilePath)
 				if err != nil {
@@ -286,6 +295,7 @@ func VariantsIngest(c echo.Context) error {
 				// -----
 
 				// --- tabix generation
+				fmt.Printf("Generating Tabix %s !\n", tmpDestinationFileName)
 				tabixFileDir, tabixFileName, tabixErr := ingestionService.GenerateTabix(tmpDestinationFileName)
 				if tabixErr != nil {
 					msg := "Something went wrong: Tabix problem " + gzippedFileName
@@ -300,6 +310,7 @@ func VariantsIngest(c echo.Context) error {
 				tabixFileNameWithRelativePath := fmt.Sprintf("%s%s", partialTmpDir, tabixFileName)
 
 				// ---   push compressed to DRS
+				fmt.Printf("Uploading %s to DRS !\n", gzippedFileName)
 				drsFileId := ingestionService.UploadVcfGzToDrs(cfg.Drs.BridgeDirectory, gzippedFileName, drsUrl, drsUsername, drsPassword)
 				if drsFileId == "" {
 					msg := "Something went wrong: DRS File Id is empty for " + gzippedFileName
@@ -313,6 +324,7 @@ func VariantsIngest(c echo.Context) error {
 				}
 
 				// -- push tabix to DRS
+				fmt.Printf("Uploading %s to DRS !\n", tabixFileNameWithRelativePath)
 				drsTabixFileId := ingestionService.UploadVcfGzToDrs(cfg.Drs.BridgeDirectory, tabixFileNameWithRelativePath, drsUrl, drsUsername, drsPassword)
 				if drsTabixFileId == "" {
 					msg := "Something went wrong: DRS Tabix File Id is empty for " + tabixFileNameWithRelativePath
@@ -326,6 +338,7 @@ func VariantsIngest(c echo.Context) error {
 				}
 
 				// ---   remove temporary files now that they have been ingested successfully into DRS
+				fmt.Printf("Removing %s !\n", tmpDestinationFileName)
 				if tmpFileRemovalErr := os.Remove(tmpDestinationFileName); tmpFileRemovalErr != nil {
 					msg := fmt.Sprintf("Something went wrong: trying to remove temporary file at %s : %s\n", tmpDestinationFileName, tmpFileRemovalErr)
 					fmt.Println(msg)
@@ -337,6 +350,7 @@ func VariantsIngest(c echo.Context) error {
 					return
 				}
 				tmpTabixFilePath := fmt.Sprintf("%s%s", tabixFileDir, tabixFileName)
+				fmt.Printf("Removing %s !\n", tmpTabixFilePath)
 				if tmpTabixFileRemovalErr := os.Remove(tmpTabixFilePath); tmpTabixFileRemovalErr != nil {
 					msg := fmt.Sprintf("Something went wrong: trying to remove temporary file at %s : %s\n", tmpTabixFilePath, tmpTabixFileRemovalErr)
 					fmt.Println(msg)
@@ -374,6 +388,7 @@ func VariantsIngest(c echo.Context) error {
 				}
 
 				// ---   extract gzip compressed vcf file
+				fmt.Printf("Extracting %s !\n", gzippedFilePath)
 				vcfFilePath := ingestionService.ExtractVcfGz(gzippedFilePath, r, vcfTmpPath)
 				if vcfFilePath == "" {
 					msg := "Something went wrong: filepath is empty for " + gzippedFileName
@@ -388,9 +403,11 @@ func VariantsIngest(c echo.Context) error {
 				defer r.Close()
 
 				// ---	 load vcf into memory and ingest the vcf file into elasticsearch
+				fmt.Printf("Begin processing %s !\n", vcfFilePath)
 				ingestionService.ProcessVcf(vcfFilePath, drsFileId, assemblyId, filterOutHomozygousReferences, cfg.Api.LineProcessingConcurrencyLevel)
 
 				// ---   delete the temporary vcf file
+				fmt.Printf("Removing temporary file %s !\n", vcfFilePath)
 				os.Remove(vcfFilePath)
 
 				// ---   delete full tmp path and all contents