Release v0.1.3

FleekHQ · Dec 11, 2020 · d3c29ae · d3c29ae
2 parents 4e3cac0 + e1e0ce9
commit d3c29ae
Show file tree

Hide file tree

Showing 94 changed files with 4,448 additions and 2,597 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -51,10 +51,9 @@ jobs:
           args: release
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          MONGO_USR: ${{ secrets.MONGO_USR }}
-          MONGO_PW: ${{ secrets.MONGO_PW }}
-          MONGO_HOST: ${{ secrets.MONGO_HOST }}
-          MONGO_REPLICA_SET: ${{ secrets.MONGO_REPLICA_SET }}
+          IPFS_ADDR: ${{ secrets.IPFS_ADDR }}
+          IPFS_NODE_ADDR: ${{ secrets.IPFS_NODE_ADDR }}
+          IPFS_NODE_PATH: ${{ secrets.IPFS_NODE_PATH }}
           SERVICES_API_URL: ${{ secrets[steps.secretnames.outputs.SERVICES_API_URL] }}
           VAULT_API_URL: ${{ secrets[steps.secretnames.outputs.VAULT_API_URL] }}
           VAULT_SALT_SECRET: ${{ secrets[steps.secretnames.outputs.VAULT_SALT_SECRET] }}

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -2,10 +2,9 @@
 on: [pull_request]
 name: Test
 jobs:
-  test:
+  unit-test:
     strategy:
       matrix:
-        #go-version: [1.13.x, 1.14.x]
         go-version: [1.14.x]
         platform: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.platform }}
@@ -16,5 +15,70 @@ jobs:
           go-version: ${{ matrix.go-version }}
       - name: Checkout code
         uses: actions/checkout@v2
-      - name: Test
-        run: go test -race ./...
+      - name: Unit Test
+        if: ${{ matrix.platform != 'windows-latest' }}
+        run: go test -race -coverprofile=coverage/unitcoverage.out $(go list ./... | grep -v integration_tests)
+      - name: Unit Test (Win)
+        if: ${{ matrix.platform == 'windows-latest' }} # skipping coverage collection on windows
+        run: go test -race $(go list ./... | grep -v integration_tests)
+      - name: Coveralls
+        if: ${{ matrix.platform != 'windows-latest' }}
+        uses: shogo82148/actions-goveralls@v1
+        with:
+          flag-name: unit-test-${{ matrix.platform }}
+          path-to-profile: coverage/unitcoverage.out
+          parallel: true
+  integration-test:
+    strategy:
+      matrix:
+        go-version: [ 1.14.x ]
+#        platform: [ ubuntu-latest, macos-latest, windows-latest ]
+        platform: [ macos-latest ]
+    runs-on: ${{ matrix.platform }}
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      SERVICES_API_URL: ${{ secrets.outputs.SERVICES_API_URL_DEV }}
+      VAULT_API_URL: ${{ secrets.VAULT_API_URL_DEV }}
+      VAULT_SALT_SECRET: ${{ secrets.VAULT_SALT_SECRET_DEV }}
+      SERVICES_HUB_AUTH_URL: ${{ secrets.SERVICES_HUB_AUTH_URL_DEV }}
+      TXL_HUB_TARGET: ${{ secrets.TXL_HUB_TARGET_DEV }}
+      TXL_HUB_MA: ${{ secrets.TXL_HUB_MA_DEV }}
+      TXL_THREADS_TARGET: ${{ secrets.TXL_THREADS_TARGET_DEV }}
+      TXL_HUB_GATEWAY_URL: ${{ secrets.TXL_HUB_GATEWAY_URL_DEV }}
+      TXL_USER_KEY: ${{ secrets.TXL_USER_KEY_DEV }}
+      TXL_USER_SECRET: ${{ secrets.TXL_USER_SECRET_DEV }}
+      SPACE_STORAGE_SITE_URL: ${{ secrets.SPACE_STORAGE_SITE_URL_DEV }}
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+      - name: Install gnome-keyring (Ubuntu)
+        if: ${{ matrix.platform == 'ubuntu-latest' }}
+        run: sudo apt-get install pass gnome-keyring dbus-x11
+      - name: Verify gnome-keyring is installed (Ubuntu)
+        if: ${{ matrix.platform == 'ubuntu-latest' }}
+        run: gnome-keyring-daemon -V
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Integration Test
+        if: ${{ matrix.platform != 'windows-latest' }}
+        run: go test -v -timeout 60m -coverprofile=coverage/integrationcoverage.out ./integration_tests/...
+      - name: Integration Test (Win)
+        if: ${{ matrix.platform == 'windows-latest' }} # skipping coverage collection on windows
+        run: go test -v -timeout 60m ./integration_tests/...
+      - name: Coveralls
+        if: ${{ matrix.platform != 'windows-latest' }}
+        uses: shogo82148/actions-goveralls@v1
+        with:
+          flag-name: integration-test-${{ matrix.platform }}
+          path-to-profile: coverage/integrationcoverage.out
+          parallel: true
+  submit-coverage:
+    needs: [unit-test, integration-test]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Coveralls Finished
+        uses: shogo82148/actions-goveralls@v1
+        with:
+          parallel-finished: true
diff --git a/.gitignore b/.gitignore
@@ -10,6 +10,8 @@
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
+coverage/*
+!coverage/.gitkeep
 
 # Dependency directories (remove the comment below to include it)
 # vendor/

diff --git a/.goreleaser.yml b/.goreleaser.yml
@@ -11,11 +11,7 @@ builds:
     # env:
     #   - CGO_ENABLED=0
     ldflags:
-      - -s -w -X main.mongousr={{ .Env.MONGO_USR }}
-      - -X main.mongopw={{ .Env.MONGO_PW }}
-      - -X main.mongohost={{ .Env.MONGO_HOST }}
-      - -X main.mongorepset={{ .Env.MONGO_REPLICA_SET }}
-      - -X main.spaceapi={{ .Env.SERVICES_API_URL }}
+      - -s -w -X  main.spaceapi={{ .Env.SERVICES_API_URL }}
       - -X main.vaultapi={{ .Env.VAULT_API_URL }}
       - -X main.vaultsaltsecret={{ .Env.VAULT_SALT_SECRET }}
       - -X main.spacehubauth={{ .Env.SERVICES_HUB_AUTH_URL }}
@@ -26,6 +22,9 @@ builds:
       - -X main.textileuserkey={{ .Env.TXL_USER_KEY }}
       - -X main.textileusersecret={{ .Env.TXL_USER_SECRET }}
       - -X main.spacestoragesiteurl={{ .Env.SPACE_STORAGE_SITE_URL }}
+      - -X main.ipfsaddr={{ .Env.IPFS_ADDR }}
+      - -X main.ipfsnodeaddr={{ .Env.IPFS_NODE_ADDR }}
+      - -X main.ipfsnodepath={{ .Env.IPFS_NODE_PATH }}
     main: ./cmd/space-daemon/main.go
     binary: space
     goos:
@@ -35,11 +34,7 @@ builds:
     # env:
     #   - CGO_ENABLED=0
     ldflags:
-      - -s -w -X main.mongousr={{ .Env.MONGO_USR }}
-      - -X main.mongopw={{ .Env.MONGO_PW }}
-      - -X main.mongohost={{ .Env.MONGO_HOST }}
-      - -X main.mongorepset={{ .Env.MONGO_REPLICA_SET }}
-      - -X main.spaceapi={{ .Env.SERVICES_API_URL }}
+      - -s -w -X main.spaceapi={{ .Env.SERVICES_API_URL }}
       - -X main.vaultapi={{ .Env.VAULT_API_URL }}
       - -X main.vaultsaltsecret={{ .Env.VAULT_SALT_SECRET }}
       - -X main.spacehubauth={{ .Env.SERVICES_HUB_AUTH_URL }}
@@ -50,6 +45,9 @@ builds:
       - -X main.textileuserkey={{ .Env.TXL_USER_KEY }}
       - -X main.textileusersecret={{ .Env.TXL_USER_SECRET }}
       - -X main.spacestoragesiteurl={{ .Env.SPACE_STORAGE_SITE_URL }}
+      - -X main.ipfsaddr={{ .Env.IPFS_ADDR }}
+      - -X main.ipfsnodeaddr={{ .Env.IPFS_NODE_ADDR }}
+      - -X main.ipfsnodepath={{ .Env.IPFS_NODE_PATH }}
     main: ./cmd/space-daemon/main.go
     binary: space
     goos:
@@ -61,11 +59,7 @@ builds:
     # env:
     #   - CGO_ENABLED=1
     ldflags:
-      - -s -w -X main.mongousr={{ .Env.MONGO_USR }}
-      - -X main.mongopw={{ .Env.MONGO_PW }}
-      - -X main.mongohost={{ .Env.MONGO_HOST }}
-      - -X main.mongorepset={{ .Env.MONGO_REPLICA_SET }}
-      - -X main.spaceapi={{ .Env.SERVICES_API_URL }}
+      - -s -w -X main.spaceapi={{ .Env.SERVICES_API_URL }}
       - -X main.vaultapi={{ .Env.VAULT_API_URL }}
       - -X main.vaultsaltsecret={{ .Env.VAULT_SALT_SECRET }}
       - -X main.spacehubauth={{ .Env.SERVICES_HUB_AUTH_URL }}
@@ -76,6 +70,9 @@ builds:
       - -X main.textileuserkey={{ .Env.TXL_USER_KEY }}
       - -X main.textileusersecret={{ .Env.TXL_USER_SECRET }}
       - -X main.spacestoragesiteurl={{ .Env.SPACE_STORAGE_SITE_URL }}
+      - -X main.ipfsaddr={{ .Env.IPFS_ADDR }}
+      - -X main.ipfsnodeaddr={{ .Env.IPFS_NODE_ADDR }}
+      - -X main.ipfsnodepath={{ .Env.IPFS_NODE_PATH }}
     main: ./cmd/space-daemon/main.go
     binary: space
     goos:

diff --git a/Makefile b/Makefile
@@ -2,10 +2,9 @@ build:
 	go build \
 	-o bin/space \
 	-ldflags \
-	"-X 'main.ipfsnodeaddr=${IPFS_NODE_ADDR}' \
+	"-X 'main.ipfsaddr=${IPFS_ADDR}' \
+	-X 'main.ipfsnodeaddr=${IPFS_NODE_ADDR}' \
 	-X 'main.ipfsnodepath=${IPFS_NODE_PATH}' \
-	-X 'main.mongousr=${MONGO_USR}' \
-	-X 'main.mongopw=${MONGO_PW}' \
 	-X 'main.spaceapi=${SERVICES_API_URL}' \
 	-X 'main.spacestoragesiteurl=${SPACE_STORAGE_SITE_URL}' \
 	-X 'main.vaultapi=${VAULT_API_URL}' \
@@ -15,27 +14,30 @@ build:
 	-X 'main.textilehubma=${TXL_HUB_MA}' \
 	-X 'main.textilethreads=${TXL_THREADS_TARGET}' \
 	-X 'main.textilehubgatewayurl=${TXL_HUB_GATEWAY_URL}' \
-  	-X 'main.textileuserkey=${TXL_USER_KEY}' \
-	-X 'main.textileusersecret=${TXL_USER_SECRET}' \
-	-X 'main.mongohost=${MONGO_HOST}' \
-	-X 'main.mongorepset=${MONGO_REPLICA_SET}'" \
+    -X 'main.textileuserkey=${TXL_USER_KEY}' \
+	-X 'main.textileusersecret=${TXL_USER_SECRET}'" \
 	cmd/space-daemon/main.go
 
 test:
-	go test ./...
+	go test $$(go list ./... | grep -v integration_tests)
+
+test_coverage:
+	go test -coverprofile=coverage/unitcoverage.out $$(go list ./... | grep -v integration_tests)
+
+integration_test:
+	go test -v -p 1 ./integration_tests/...
+
+integration_test_coverage:
+	go test -v -p 1 -coverprofile=coverage/integrationcoverage.out ./integration_tests/...
 
 proto_gen:
 	protoc -I grpc/pb/ -I grpc/proto/ -I./devtools/googleapis grpc/proto/space.proto --go_out=plugins=grpc:grpc/pb
 
 gen_rest:
 	protoc -I grpc/pb/ -I grpc/proto/ -I./devtools/googleapis grpc/proto/space.proto --go_out=plugins=grpc:grpc/pb --grpc-gateway_out=logtostderr=true:grpc/pb
 
-## this target requires both protoc-gen-swagger and statik to be installed
-gen_rest_swagger:
-	protoc -I grpc/pb/ -I grpc/proto/ -I./devtools/googleapis grpc/proto/space.proto --swagger_out=logtostderr=true:swagger/ui \
-		&& statik -src=swagger/ui -f -dest=swagger -p=bin_ui
 
-gen_all: proto_gen gen_rest gen_rest_swagger
+gen_all: proto_gen gen_rest
 
 ## runs jaeger tracing server, should be used when trace is enabled on daemon
 jaegar:

diff --git a/README.md b/README.md
@@ -76,20 +76,12 @@ These are centralized services that are optional, but offer additional convenien
 
 Our hosted Textile Hub requires authentication via public key for logging in. This service sends a challenge to Space Daemon, which signs the challenge with the private key of the user and in that way our hosted Textile Hub can allow the user to store data.
 
-### MongoDB
-
-Currently, local Textile Threads require a running MongoDB database. Space Daemon by default connects to a hosted one, but this will be removed once Textile switches over to an embeddable data store.
-
 ## Running from source
 
 After cloning this repo, you can run it from source by running `go run ./cmd/space-daemon -dev`. Consider that you will need the following environment variables exported in your system:
 
 ```
 IPFS_ADDR=[Your IPFS node address]
-MONGO_PW=[The password of a MongoDB database]
-MONGO_USR=[The user of a MongoDB database]
-MONGO_HOST=[The host of a MongoDB database]
-MONGO_REPLICA_SET=[The replica set for a MongoDB database]
 SERVICES_API_URL=[The URL where Space Services API is located]
 VAULT_API_URL=[The URL where Space Vault API is located]
 VAULT_SALT_SECRET=[A random string used for kdf functions before storing keys to the vault]
@@ -149,26 +141,17 @@ If you update the gRPC API, you need to regenerate the Protobuf file.
 You will need to install the following binaries in your Go path:
 
 - `go get -u github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway`
-- `go get -u github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger`
-
-If you don't have swagger generator, you will need it for generating the REST API documentation:
-
-`brew install swagger-codegen`
-`brew install statik`
 
 Checking the binaries:
 `ls $GOPATH/bin`
-Should show the following binaries in your path: protoc-gen-go, protoc-gen-grpc-gateway, protoc-gen-swagger
+Should show the following binaries in your path: protoc-gen-go, protoc-gen-grpc-gateway
 
 Run the protobuf generation:
 `make proto_gen`
 
 Run the REST proxy generation:
 `make gen_rest`
 
-To generate REST proxy swagger spec and ui binary generation
-`make gen_rest_swagger`
-
 ** Ideally you should run `make gen_all` before commiting as this would run all the above three code generations and
 ensure everything is up to date **
 
@@ -197,7 +180,7 @@ Secrets are set by adding them in Github and then specifying them in `release.ym
 
 If specified, the release file will dynamically generate the secret name based on the stage by adding a `_DEV` or `_PRD` suffix to the secret name only for the specificed environment variable. It will always use `_PRD` unless the tag ends in `-dev`.  So for example tag `v0.0.15` will use PRD values, while `v0.0.15-dev` will use DEV values.
 
-Stage specific secret names will only be used for secrets `release.yml` that point to the step output instead of the secret name directly (i.e., `SERVICES_API_URL: ${{ secrets[steps.secretnames.outputs.SERVICES_API_URL] }}` instead of `MONGO_REPLICA_SET: ${{ secrets.MONGO_REPLICA_SET }}`.
+Stage specific secret names will only be used for secrets in `release.yml` that point to the step output instead of the secret name directly (i.e., `SERVICES_API_URL: ${{ secrets[steps.secretnames.outputs.SERVICES_API_URL] }}` instead of `SERVICES_API_URL: ${{ secrets.SERVICES_API_URL }}`.
 
 So to add a new secret:
 * If it's not stage specific then add the secret in GH with no suffix and in `release.yml`, refer to it based on the secret name.