diff --git a/.gitignore b/.gitignore index 1115f4e7..1f8e0c5c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,8 +6,8 @@ .env # Volume directories -data -data-* +data/ +data-*/ # vcfs */vcfs/*.txt diff --git a/Makefile b/Makefile index 2d03a672..d1cccd31 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ build-gateway: stop-gateway clean-gateway echo "-- Building Gateway Container --" docker-compose -f docker-compose.yaml build gateway -build-api-binaries: +build-api-local-binaries: @echo "-- Building Golang-Api-Alpine Binaries --" cd src/api && \ @@ -55,11 +55,9 @@ build-api-binaries: cd ../.. && \ upx --brute bin/api_${GOOS}_${GOARCH} -build-api-container: stop-api clean-api build-api-binaries +build-api: stop-api clean-api @echo "-- Building Golang-Api-Alpine Container --" - cp bin/api_${GOOS}_${GOARCH} src/api docker-compose -f docker-compose.yaml build api - rm src/api/api_${GOOS}_${GOARCH} build-drs: stop-drs clean-drs @echo "-- Building DRS Container --" diff --git a/README.md b/README.md index b8da9843..ee5dda33 100644 --- a/README.md +++ b/README.md @@ -390,12 +390,12 @@ Local Release:  From the project root, run ``` -make build-api-binaries +make build-api-local-binaries ``` - The binary can then be found at *bin/api_${GOOS}_${GOARCH}* and executed with + The binary can then be found at *bin/api_${GOOS}_${GOARCH}* and executed locally with ``` export GOHAN_API_INTERNAL_PORT=${GOHAN_API_INTERNAL_PORT} @@ -426,10 +426,10 @@ cd bin/ Containerized Alpine Release: - When ready, build the `docker image` and spawn the `container` by running + When ready, build the `docker image` and spawn the `container` with an independent binary build by running ``` -make build-api-container +make build-api make run-api ``` diff --git a/docker-compose.yaml b/docker-compose.yaml index 066bb44e..a4674465 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -21,8 +21,8 @@ services: build: context: $PWD/src/api args: + BUILDER_BASE_IMAGE: ${GOHAN_API_BUILDER_BASE_IMAGE} BASE_IMAGE: ${GOHAN_API_BASE_IMAGE} - API_EXECUTABLE_FILENAME: api_linux_amd64 # also passed in as an ENV from within Dockerfile : GOHAN_API_INTERNAL_PORT: ${GOHAN_API_INTERNAL_PORT} image: ${GOHAN_API_IMAGE}:${GOHAN_API_VERSION} diff --git a/etc/example.env b/etc/example.env index d10aa3d5..5c766cec 100644 --- a/etc/example.env +++ b/etc/example.env @@ -27,6 +27,7 @@ GOHAN_GATEWAY_CONTAINER_NAME=gohan-gateway GOHAN_API_IMAGE=gohan-api GOHAN_API_VERSION=latest +GOHAN_API_BUILDER_BASE_IMAGE=golang:1.15.15-alpine3.14 GOHAN_API_BASE_IMAGE=alpine GOHAN_API_CONTAINER_NAME=gohan-api diff --git a/src/api/Dockerfile b/src/api/Dockerfile index e38f4e8b..4aac3d79 100644 --- a/src/api/Dockerfile +++ b/src/api/Dockerfile @@ -1,22 +1,38 @@ +ARG BUILDER_BASE_IMAGE ARG BASE_IMAGE -FROM $BASE_IMAGE + +# Stage 1 - builder +FROM $BUILDER_BASE_IMAGE as builder # Maintainer LABEL maintainer="Brennan Brouillette " +WORKDIR /build + +COPY . /build/ + # Alpine updates and executable dependencies -RUN apk update \ - && apk upgrade +RUN apk update && \ + apk upgrade && \ + apk add git upx + +RUN CGO_ENABLED=0 && \ + go build -ldflags="-s -w" -o gohan_api && \ + upx --brute gohan_api + + +# Stage two - executioner +FROM $BASE_IMAGE + +# # Alpine updates +RUN apk update && \ + apk upgrade # Copy pre-built executeables -ARG API_EXECUTABLE_FILENAME -ENV api_executable_filename=$API_EXECUTABLE_FILENAME -COPY $API_EXECUTABLE_FILENAME /app/ +COPY --from=builder /build/gohan_api /app/ +# Run ARG GOHAN_API_INTERNAL_PORT ENV GOHAN_API_INTERNAL_PORT=$GOHAN_API_INTERNAL_PORT EXPOSE $GOHAN_API_INTERNAL_PORT - -# Run -WORKDIR /app -ENTRYPOINT ./$api_executable_filename +ENTRYPOINT [ "/app/gohan_api" ] diff --git a/src/api/contexts/contexts.go b/src/api/contexts/contexts.go index f6a47811..c004d1cc 100644 --- a/src/api/contexts/contexts.go +++ b/src/api/contexts/contexts.go @@ -4,7 +4,7 @@ import ( "api/models" "api/services" - es7 "github.com/elastic/go-elasticsearch" + es7 "github.com/elastic/go-elasticsearch/v7" "github.com/labstack/echo" ) diff --git a/src/api/go.mod b/src/api/go.mod new file mode 100644 index 00000000..bc2af911 --- /dev/null +++ b/src/api/go.mod @@ -0,0 +1,17 @@ +module api + +go 1.15 + +require ( + github.com/Jeffail/gabs v1.4.0 // indirect + github.com/cenkalti/backoff v2.2.1+incompatible // indirect + github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect + github.com/elastic/go-elasticsearch/v7 v7.15.0 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/kelseyhightower/envconfig v1.4.0 // indirect + github.com/labstack/echo v3.3.10+incompatible // indirect + github.com/labstack/gommon v0.3.0 // indirect + github.com/mitchellh/mapstructure v1.4.2 // indirect + github.com/valyala/fasttemplate v1.2.1 // indirect + golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect +) diff --git a/src/api/go.sum b/src/api/go.sum new file mode 100644 index 00000000..4e158fee --- /dev/null +++ b/src/api/go.sum @@ -0,0 +1,51 @@ +github.com/Jeffail/gabs v1.4.0 h1://5fYRRTq1edjfIrQGvdkcd22pkYUrHZ5YC/H2GJVAo= +github.com/Jeffail/gabs v1.4.0/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc= +github.com/cenkalti/backoff v1.1.0 h1:QnvVp8ikKCDWOsFheytRCoYWYPO/ObCTBGxT19Hc+yE= +github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= +github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v1.0.2 h1:KPldsxuKGsS2FPWsNeg9ZO18aCrGKujPoWXn2yo+KQM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/elastic/go-elasticsearch v0.0.0 h1:Pd5fqOuBxKxv83b0+xOAJDAkziWYwFinWnBO0y+TZaA= +github.com/elastic/go-elasticsearch/v7 v7.15.0 h1:kMLoRAO97gAV97YKgn2z/5ExM8pVJeLoJPR0r33OREs= +github.com/elastic/go-elasticsearch/v7 v7.15.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/labstack/echo v1.4.4 h1:1bEiBNeGSUKxcPDGfZ/7IgdhJJZx8wV/pICJh4W2NJI= +github.com/labstack/echo v3.3.10+incompatible h1:pGRcYk231ExFAyoAjAfD85kQzRJCRI8bbnE7CX5OEgg= +github.com/labstack/echo v3.3.10+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8cVwCLbBmJyDaka6Cmk1s= +github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo= +github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/src/api/mvc/data-types.go b/src/api/mvc/data-types.go new file mode 100644 index 00000000..92712161 --- /dev/null +++ b/src/api/mvc/data-types.go @@ -0,0 +1,36 @@ +package mvc + +import ( + "net/http" + + "api/models/schemas" + + "github.com/labstack/echo" +) + +var variantDataTypeJson = map[string]interface{}{ + "id": "variant", + "schema": schemas.VARIANT_SCHEMA, + "metadata_schema": schemas.VARIANT_TABLE_METADATA_SCHEMA, +} + +func GetDataTypes(c echo.Context) error { + // Data types are basically stand-ins for schema blocks + return c.JSON(http.StatusOK, []map[string]interface{}{ + variantDataTypeJson, + }) +} + +func GetVariantDataType(c echo.Context) error { + return c.JSON(http.StatusOK, variantDataTypeJson) +} + +func GetVariantDataTypeSchema(c echo.Context) error { + return c.JSON(http.StatusOK, schemas.VARIANT_SCHEMA) +} + +func GetVariantDataTypeMetadataSchema(c echo.Context) error { + return c.JSON(http.StatusOK, schemas.VARIANT_TABLE_METADATA_SCHEMA) +} + +// # TODO: Consistent snake or kebab diff --git a/src/api/mvc/variants.go b/src/api/mvc/variants.go index 3d17e08d..d2b09b93 100644 --- a/src/api/mvc/variants.go +++ b/src/api/mvc/variants.go @@ -23,7 +23,7 @@ import ( esRepo "api/repositories/elasticsearch" "api/utils" - "github.com/elastic/go-elasticsearch" + "github.com/elastic/go-elasticsearch/v7" "github.com/google/uuid" "github.com/labstack/echo" diff --git a/src/api/repositories/elasticsearch/genes.go b/src/api/repositories/elasticsearch/genes.go index d7133ef8..5f66432b 100644 --- a/src/api/repositories/elasticsearch/genes.go +++ b/src/api/repositories/elasticsearch/genes.go @@ -14,7 +14,7 @@ import ( "api/models/constants" assemblyId "api/models/constants/assembly-id" - "github.com/elastic/go-elasticsearch" + "github.com/elastic/go-elasticsearch/v7" ) const genesIndex = "genes" diff --git a/src/api/repositories/elasticsearch/variants.go b/src/api/repositories/elasticsearch/variants.go index 7da69be4..eaef86bc 100644 --- a/src/api/repositories/elasticsearch/variants.go +++ b/src/api/repositories/elasticsearch/variants.go @@ -16,7 +16,7 @@ import ( s "api/models/constants/sort" z "api/models/constants/zygosity" - "github.com/elastic/go-elasticsearch" + "github.com/elastic/go-elasticsearch/v7" ) const variantsIndex = "variants" diff --git a/src/api/services/ingestion.go b/src/api/services/ingestion.go index 252786a5..9c827226 100644 --- a/src/api/services/ingestion.go +++ b/src/api/services/ingestion.go @@ -28,8 +28,8 @@ import ( "time" "github.com/Jeffail/gabs" - "github.com/elastic/go-elasticsearch" - "github.com/elastic/go-elasticsearch/esutil" + "github.com/elastic/go-elasticsearch/v7" + "github.com/elastic/go-elasticsearch/v7/esutil" "github.com/mitchellh/mapstructure" ) diff --git a/src/api/utils/connections.go b/src/api/utils/connections.go index 1e350cec..02399e82 100644 --- a/src/api/utils/connections.go +++ b/src/api/utils/connections.go @@ -7,7 +7,7 @@ import ( "time" "github.com/cenkalti/backoff" - es7 "github.com/elastic/go-elasticsearch" + es7 "github.com/elastic/go-elasticsearch/v7" ) func CreateEsConnection(cfg *models.Config) *es7.Client {