Merge pull request #13 from bento-platform/qa/v3.0.0

v3.0.2 candidate

.gitignore

@@ -6,8 +6,8 @@
 .env
 
 # Volume directories
-data
-data-*
+data/
+data-*/
 
 # vcfs
 */vcfs/*.txt

Makefile

@@ -44,7 +44,7 @@ build-gateway: stop-gateway clean-gateway
 	echo "-- Building Gateway Container --"
 	docker-compose -f docker-compose.yaml build gateway
 
-build-api-binaries:
+build-api-local-binaries:
 	@echo "-- Building Golang-Api-Alpine Binaries --"
 
 	cd src/api && \
@@ -55,11 +55,9 @@ build-api-binaries:
 	cd ../.. && \
 	upx --brute bin/api_${GOOS}_${GOARCH}
 
-build-api-container: stop-api clean-api build-api-binaries
+build-api: stop-api clean-api
 	@echo "-- Building Golang-Api-Alpine Container --"
-	cp bin/api_${GOOS}_${GOARCH} src/api
 	docker-compose -f docker-compose.yaml build api
-	rm src/api/api_${GOOS}_${GOARCH}
 
 build-drs: stop-drs clean-drs
 	@echo "-- Building DRS Container --"

README.md

@@ -390,12 +390,12 @@ Local Release:
  From the project root, run 
 
 ```
-make build-api-binaries
+make build-api-local-binaries
 ```
 
 
 
- The binary can then be found at *bin/api_${GOOS}_${GOARCH}* and executed with
+ The binary can then be found at *bin/api_${GOOS}_${GOARCH}* and executed locally with
 
 ```
 export GOHAN_API_INTERNAL_PORT=${GOHAN_API_INTERNAL_PORT}
@@ -426,10 +426,10 @@ cd bin/
 
 Containerized Alpine Release: 
 
- When ready, build the `docker image` and spawn the `container` by running
+ When ready, build the `docker image` and spawn the `container` with an independent binary build by running
 
 ```
-make build-api-container
+make build-api
 make run-api
 ```
 

docker-compose.yaml

@@ -21,8 +21,8 @@ services:
     build:
       context: $PWD/src/api
       args:
+        BUILDER_BASE_IMAGE: ${GOHAN_API_BUILDER_BASE_IMAGE}
         BASE_IMAGE: ${GOHAN_API_BASE_IMAGE}
-        API_EXECUTABLE_FILENAME: api_linux_amd64
         # also passed in as an ENV from within Dockerfile :
         GOHAN_API_INTERNAL_PORT: ${GOHAN_API_INTERNAL_PORT} 
     image: ${GOHAN_API_IMAGE}:${GOHAN_API_VERSION}

etc/example.env

@@ -27,6 +27,7 @@ GOHAN_GATEWAY_CONTAINER_NAME=gohan-gateway
 GOHAN_API_IMAGE=gohan-api
 GOHAN_API_VERSION=latest
 
+GOHAN_API_BUILDER_BASE_IMAGE=golang:1.15.15-alpine3.14
 GOHAN_API_BASE_IMAGE=alpine
 
 GOHAN_API_CONTAINER_NAME=gohan-api

src/api/Dockerfile

@@ -1,22 +1,38 @@
+ARG BUILDER_BASE_IMAGE
 ARG BASE_IMAGE
-FROM $BASE_IMAGE
+
+# Stage 1 - builder
+FROM $BUILDER_BASE_IMAGE as builder
 
 # Maintainer
 LABEL maintainer="Brennan Brouillette <[email protected]>"
 
+WORKDIR /build
+
+COPY . /build/
+
 # Alpine updates and executable dependencies
-RUN apk update \
-    && apk upgrade
+RUN apk update && \
+    apk upgrade && \
+    apk add git upx
+
+RUN CGO_ENABLED=0 && \
+    go build -ldflags="-s -w" -o gohan_api && \
+    upx --brute gohan_api
+
+
+# Stage two - executioner
+FROM $BASE_IMAGE
+
+# # Alpine updates
+RUN apk update && \
+    apk upgrade
 
 # Copy pre-built executeables
-ARG API_EXECUTABLE_FILENAME
-ENV api_executable_filename=$API_EXECUTABLE_FILENAME
-COPY $API_EXECUTABLE_FILENAME /app/
+COPY --from=builder /build/gohan_api /app/
 
+# Run
 ARG GOHAN_API_INTERNAL_PORT
 ENV GOHAN_API_INTERNAL_PORT=$GOHAN_API_INTERNAL_PORT
 EXPOSE $GOHAN_API_INTERNAL_PORT
-
-# Run
-WORKDIR /app
-ENTRYPOINT ./$api_executable_filename
+ENTRYPOINT [ "/app/gohan_api" ]

src/api/contexts/contexts.go

@@ -4,7 +4,7 @@ import (
 	"api/models"
 	"api/services"
 
-	es7 "github.com/elastic/go-elasticsearch"
+	es7 "github.com/elastic/go-elasticsearch/v7"
 	"github.com/labstack/echo"
 )
 

src/api/go.mod

@@ -0,0 +1,17 @@
+module api
+
+go 1.15
+
+require (
+	github.com/Jeffail/gabs v1.4.0 // indirect
+	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
+	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
+	github.com/elastic/go-elasticsearch/v7 v7.15.0 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/kelseyhightower/envconfig v1.4.0 // indirect
+	github.com/labstack/echo v3.3.10+incompatible // indirect
+	github.com/labstack/gommon v0.3.0 // indirect
+	github.com/mitchellh/mapstructure v1.4.2 // indirect
+	github.com/valyala/fasttemplate v1.2.1 // indirect
+	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
+)

src/api/go.sum

@@ -0,0 +1,51 @@
+github.com/Jeffail/gabs v1.4.0 h1://5fYRRTq1edjfIrQGvdkcd22pkYUrHZ5YC/H2GJVAo=
+github.com/Jeffail/gabs v1.4.0/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc=
+github.com/cenkalti/backoff v1.1.0 h1:QnvVp8ikKCDWOsFheytRCoYWYPO/ObCTBGxT19Hc+yE=
+github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
+github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v1.0.2 h1:KPldsxuKGsS2FPWsNeg9ZO18aCrGKujPoWXn2yo+KQM=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/elastic/go-elasticsearch v0.0.0 h1:Pd5fqOuBxKxv83b0+xOAJDAkziWYwFinWnBO0y+TZaA=
+github.com/elastic/go-elasticsearch/v7 v7.15.0 h1:kMLoRAO97gAV97YKgn2z/5ExM8pVJeLoJPR0r33OREs=
+github.com/elastic/go-elasticsearch/v7 v7.15.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
+github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
+github.com/labstack/echo v1.4.4 h1:1bEiBNeGSUKxcPDGfZ/7IgdhJJZx8wV/pICJh4W2NJI=
+github.com/labstack/echo v3.3.10+incompatible h1:pGRcYk231ExFAyoAjAfD85kQzRJCRI8bbnE7CX5OEgg=
+github.com/labstack/echo v3.3.10+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8cVwCLbBmJyDaka6Cmk1s=
+github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0=
+github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
+github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
+github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=
+github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
+github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo=
+github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
+github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4=
+github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

src/api/mvc/data-types.go

@@ -0,0 +1,36 @@
+package mvc
+
+import (
+	"net/http"
+
+	"api/models/schemas"
+
+	"github.com/labstack/echo"
+)
+
+var variantDataTypeJson = map[string]interface{}{
+	"id":              "variant",
+	"schema":          schemas.VARIANT_SCHEMA,
+	"metadata_schema": schemas.VARIANT_TABLE_METADATA_SCHEMA,
+}
+
+func GetDataTypes(c echo.Context) error {
+	// Data types are basically stand-ins for schema blocks
+	return c.JSON(http.StatusOK, []map[string]interface{}{
+		variantDataTypeJson,
+	})
+}
+
+func GetVariantDataType(c echo.Context) error {
+	return c.JSON(http.StatusOK, variantDataTypeJson)
+}
+
+func GetVariantDataTypeSchema(c echo.Context) error {
+	return c.JSON(http.StatusOK, schemas.VARIANT_SCHEMA)
+}
+
+func GetVariantDataTypeMetadataSchema(c echo.Context) error {
+	return c.JSON(http.StatusOK, schemas.VARIANT_TABLE_METADATA_SCHEMA)
+}
+
+// # TODO: Consistent snake or kebab

src/api/mvc/variants.go

@@ -23,7 +23,7 @@ import (
 	esRepo "api/repositories/elasticsearch"
 	"api/utils"
 
-	"github.com/elastic/go-elasticsearch"
+	"github.com/elastic/go-elasticsearch/v7"
 	"github.com/google/uuid"
 
 	"github.com/labstack/echo"

src/api/repositories/elasticsearch/genes.go

@@ -14,7 +14,7 @@ import (
 	"api/models/constants"
 	assemblyId "api/models/constants/assembly-id"
 
-	"github.com/elastic/go-elasticsearch"
+	"github.com/elastic/go-elasticsearch/v7"
 )
 
 const genesIndex = "genes"

src/api/repositories/elasticsearch/variants.go

@@ -16,7 +16,7 @@ import (
 	s "api/models/constants/sort"
 	z "api/models/constants/zygosity"
 
-	"github.com/elastic/go-elasticsearch"
+	"github.com/elastic/go-elasticsearch/v7"
 )
 
 const variantsIndex = "variants"

src/api/services/ingestion.go

@@ -28,8 +28,8 @@ import (
 	"time"
 
 	"github.com/Jeffail/gabs"
-	"github.com/elastic/go-elasticsearch"
-	"github.com/elastic/go-elasticsearch/esutil"
+	"github.com/elastic/go-elasticsearch/v7"
+	"github.com/elastic/go-elasticsearch/v7/esutil"
 	"github.com/mitchellh/mapstructure"
 )
 

src/api/utils/connections.go

@@ -7,7 +7,7 @@ import (
 	"time"
 
 	"github.com/cenkalti/backoff"
-	es7 "github.com/elastic/go-elasticsearch"
+	es7 "github.com/elastic/go-elasticsearch/v7"
 )
 
 func CreateEsConnection(cfg *models.Config) *es7.Client {