Feat: Feast go support with HTTP Registry (#33)

* Feat: Feast go support with HTTP Registry
* fixed feast version issue
* removed commented code
* HTTP Registry reads metadata specific to project instead of all projects
---------

Co-authored-by: Bhargav Dodla <[email protected]>

.github/workflows/linter.yml

@@ -15,6 +15,11 @@ jobs:
         with:
           python-version: "3.8"
           architecture: x64
+      - name: Setup Go
+        id: setup-go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.19.7
       - name: Upgrade pip version
         run: |
           pip install --upgrade pip
@@ -34,8 +39,44 @@ jobs:
             ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
       - name: Install pip-tools
         run: pip install pip-tools
+      - name: Install apache-arrow on ubuntu
+        run: |
+          sudo apt update
+          sudo apt install -y -V ca-certificates lsb-release wget
+          wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+          sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+          sudo apt update
+          sudo apt install -y -V "libarrow-dev=11.0.0-1"
       - name: Install dependencies
         run: |
+          make compile-protos-go
           make install-python-ci-dependencies
       - name: Lint python
         run: make lint-python
+  lint-go:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Go
+        id: setup-go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.19.7
+      - name: Setup Python
+        id: setup-python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+      - name: Upgrade pip version
+        run: |
+          pip install --upgrade pip
+      - name: Install apache-arrow on ubuntu
+        run: |
+            sudo apt update
+            sudo apt install -y -V ca-certificates lsb-release wget
+            wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+            sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+            sudo apt update
+            sudo apt install -y -V "libarrow-dev=11.0.0-1"
+      - name: Lint go
+        run: make lint-go

.github/workflows/pr_integration_tests.yml

@@ -115,6 +115,11 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           architecture: x64
+      - name: Setup Go
+        id: setup-go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.19.7
       - name: Authenticate to Google Cloud
         uses: 'google-github-actions/auth@v1'
         with:
@@ -152,6 +157,20 @@ jobs:
             ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
       - name: Install pip-tools
         run: pip install pip-tools
+      - name: Install apache-arrow on ubuntu
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+            sudo apt update
+            sudo apt install -y -V ca-certificates lsb-release wget
+            wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+            sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+            sudo apt update
+            sudo apt install -y -V "libarrow-dev=11.0.0-1"
+      - name: Install apache-arrow on macos
+        if: matrix.os == 'macOS-latest'
+        run: |
+            brew install apache-arrow
+            brew install pkg-config
       - name: Install dependencies
         run: make install-python-ci-dependencies
       - name: Setup Redis Cluster

.github/workflows/pr_local_integration_tests.yml

@@ -38,13 +38,22 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           architecture: x64
+
+      - name: Setup Go
+        id: setup-go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.19.7
+
       - name: Upgrade pip version
         run: |
           pip install --upgrade pip
+
       - name: Get pip cache dir
         id: pip-cache
         run: |
           echo "::set-output name=dir::$(pip cache dir)"
+
       - name: pip cache
         uses: actions/cache@v2
         with:
@@ -55,10 +64,23 @@ jobs:
           key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
           restore-keys: |
             ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
+
       - name: Install pip-tools
         run: pip install pip-tools
+
+      - name: Install apache-arrow on ubuntu
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+            sudo apt update
+            sudo apt install -y -V ca-certificates lsb-release wget
+            wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+            sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+            sudo apt update
+            sudo apt install -y -V "libarrow-dev=11.0.0-1"
+
       - name: Install dependencies
         run: make install-python-ci-dependencies
+
       - name: Test local integration tests
         if: ${{ always() }}  # this will guarantee that step won't be canceled and resources won't leak
         env:

.github/workflows/unit_tests.yml

@@ -27,6 +27,11 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           architecture: x64
+      - name: Setup Go
+        id: setup-go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.19.7
       - name: Install mysql on macOS
         if: startsWith(matrix.os, 'macOS')
         run: |
@@ -58,11 +63,54 @@ jobs:
       - name: Install pip-tools
         run: |
           pip install -U pip-tools
+      - name: Install apache-arrow on ubuntu
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+            sudo apt update
+            sudo apt install -y -V ca-certificates lsb-release wget
+            wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+            sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+            sudo apt update
+            sudo apt install -y -V "libarrow-dev=11.0.0-1"
+      - name: Install apache-arrow on macos
+        if: matrix.os == 'macOS-latest'
+        run: |
+            brew install apache-arrow
+            brew install pkg-config
       - name: Install dependencies
         run: make install-python-ci-dependencies
       - name: Test Python
         run: pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests
 
+  unit-test-go:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Python
+        id: setup-python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+      - name: Upgrade pip version
+        run: |
+          pip install --upgrade "pip>=22.1,<23"
+      - name: Setup Go
+        id: setup-go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.19.7
+      - name: Install apache-arrow on ubuntu
+        run: |
+            sudo apt update
+            sudo apt install -y -V ca-certificates lsb-release wget
+            wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+            sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+            sudo apt update
+            sudo apt install -y -V "libarrow-dev=11.0.0-1"
+            sudo apt install -y -V pkg-config
+      - name: Test
+        run: make test-go
+
   unit-test-ui:
     runs-on: ubuntu-latest
     env:

MANIFEST.in

@@ -6,3 +6,4 @@ prune infra
 prune examples
 
 graft sdk/python/feast/ui/build
+graft sdk/python/feast/embedded_go/lib

Makefile

@@ -24,19 +24,19 @@ TRINO_VERSION ?= 376
 
 # General
 
-format: format-python format-java
+format: format-python format-java format-go
 
-lint: lint-python lint-java
+lint: lint-python lint-java lint-go
 
-test: test-python test-java
+test: test-python test-java test-go
 
-protos: compile-protos-python compile-protos-docs
+protos: compile-protos-go compile-protos-python compile-protos-docs
 
 build: protos build-java build-docker
 
 # Python SDK
 
-install-python-ci-dependencies:
+install-python-ci-dependencies: install-go-proto-dependencies install-go-ci-dependencies
 	python -m piptools sync sdk/python/requirements/py$(PYTHON)-ci-requirements.txt
 	COMPILE_GO=true python setup.py develop
 
@@ -313,6 +313,9 @@ test-python-universal-cassandra-no-cloud-providers:
 test-python-universal:
 	FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests
 
+test-python-go-server: compile-go-lib
+ 	FEAST_USAGE=False IS_TEST=True pytest --integration --goserver sdk/python/tests
+
 format-python:
 	# Sort
 	cd ${ROOT_DIR}/sdk/python; python -m isort feast/ tests/
@@ -363,12 +366,45 @@ test-trino-plugin-locally:
 kill-trino-locally:
 	cd ${ROOT_DIR}; docker stop trino
 
+# Go SDK & embedded
+
+install-go-proto-dependencies:
+	go install google.golang.org/protobuf/cmd/[email protected]
+	go install google.golang.org/grpc/cmd/[email protected]
+
+install-go-ci-dependencies:
+	# TODO: currently gopy installation doesn't work w/o explicit go get in the next line
+	# TODO: there should be a better way to install gopy
+	go get github.com/go-python/[email protected]
+	go install golang.org/x/tools/cmd/goimports
+	# The `go get` command on the previous lines download the lib along with replacing the dep to `feast-dev/gopy`
+	# but the following command is needed to install it for some reason.
+	go install github.com/go-python/gopy
+	python -m pip install "pybindgen==0.22.1" "protobuf<5,>3.20"
+
 install-protoc-dependencies:
-	pip install --ignore-installed protobuf grpcio-tools==1.47.0 mypy-protobuf==3.1.0
+	pip install "protobuf<5,>3.20" "grpcio-tools>=1.47.0" "mypy-protobuf==3.1"
+
+compile-protos-go: install-go-proto-dependencies install-protoc-dependencies
+	python setup.py build_go_protos
+
+compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies
+	CGO_LDFLAGS_ALLOW=".*" COMPILE_GO=True python setup.py build_ext --inplace
 
 install-feast-ci-locally:
 	pip install -e ".[ci]"
 
+# Needs feast package to setup the feature store
+# CGO flag is due to this issue: https://github.com/golang/go/wiki/InvalidFlag
+test-go: compile-protos-go compile-protos-python  compile-go-lib install-feast-ci-locally
+	CGO_LDFLAGS_ALLOW=".*" go test -tags cgo,ccalloc ./...
+
+format-go:
+	gofmt -s -w go/
+
+lint-go: compile-protos-go compile-go-lib
+	go vet -tags cgo,ccalloc ./go/internal/feast ./go/embedded
+
 # Docker
 
 build-docker: build-feature-server-python-aws-docker build-feature-transformation-server-docker build-feature-server-java-docker
@@ -430,7 +466,7 @@ install-dependencies-proto-docs:
 	cd ${ROOT_DIR}/protos;
 	mkdir -p $$HOME/bin
 	mkdir -p $$HOME/include
-	go get github.com/golang/protobuf/proto && \
+	go get google.golang.org/protobuf/proto && \
 	go get github.com/russross/blackfriday/v2 && \
 	cd $$(mktemp -d) && \
 	git clone https://github.com/istio/tools/ && \