Merge pull request #199 from FleekHQ/alpha-release-dev

Release: alpha to master

.github/workflows/release.yml

@@ -13,7 +13,7 @@ jobs:
         run: git fetch --prune --unshallow
       - uses: actions/setup-go@v2-beta
         with:
-          go-version: '^1.14.0'
+          go-version: 1.14
       - name: Add MacOS certs
         run: cp ./ci/add-osx-cert.sh /tmp/add-osx-cert.sh && chmod +x /tmp/add-osx-cert.sh && /tmp/add-osx-cert.sh
         env:
@@ -23,6 +23,27 @@ jobs:
         run: |
           brew tap mitchellh/gon
           brew install mitchellh/gon/gon
+      - name: Set stage to prd for all
+        run: |
+          echo "::set-env name=STAGE::PRD"
+      - name: Set stage to dev if tagged develop
+        if: endsWith(github.ref, '-dev')
+        run: |
+          echo "::set-env name=STAGE::DEV"
+      - name: Set secret names
+        id: secretnames
+        run: |
+          echo $STAGE
+          echo "::set-output name=SERVICES_API_URL::SERVICES_API_URL_${STAGE}"
+          echo "::set-output name=VAULT_API_URL::VAULT_API_URL_${STAGE}"
+          echo "::set-output name=VAULT_SALT_SECRET::VAULT_SALT_SECRET_${STAGE}"
+          echo "::set-output name=SERVICES_HUB_AUTH_URL::SERVICES_HUB_AUTH_URL_${STAGE}"
+          echo "::set-output name=TXL_HUB_TARGET::TXL_HUB_TARGET_${STAGE}"
+          echo "::set-output name=TXL_HUB_MA::TXL_HUB_MA_${STAGE}"
+          echo "::set-output name=TXL_THREADS_TARGET::TXL_THREADS_TARGET_${STAGE}"
+          echo "::set-output name=TXL_HUB_GATEWAY_URL::TXL_HUB_GATEWAY_URL_${STAGE}"
+          echo "::set-output name=TXL_USER_KEY::TXL_USER_KEY_${STAGE}"
+          echo "::set-output name=TXL_USER_SECRET::TXL_USER_SECRET_${STAGE}"
       - name: Release via goreleaser
         uses: goreleaser/goreleaser-action@master
         with:
@@ -33,12 +54,13 @@ jobs:
           MONGO_PW: ${{ secrets.MONGO_PW }}
           MONGO_HOST: ${{ secrets.MONGO_HOST }}
           MONGO_REPLICA_SET: ${{ secrets.MONGO_REPLICA_SET }}
-          SERVICES_API_URL: ${{ secrets.SERVICES_API_URL }}
-          VAULT_API_URL: ${{ secrets.VAULT_API_URL }}
-          VAULT_SALT_SECRET: ${{ secrets.VAULT_SALT_SECRET }}
-          SERVICES_HUB_AUTH_URL: ${{ secrets.SERVICES_HUB_AUTH_URL }}
-          TXL_HUB_TARGET: ${{ secrets.TXL_HUB_TARGET }}
-          TXL_HUB_MA: ${{ secrets.TXL_HUB_MA }}
-          TXL_THREADS_TARGET: ${{ secrets.TXL_THREADS_TARGET }}
-          TXL_USER_KEY: ${{ secrets.TXL_USER_KEY }}
-          TXL_USER_SECRET: ${{ secrets.TXL_USER_SECRET }}
+          SERVICES_API_URL: ${{ secrets[steps.secretnames.outputs.SERVICES_API_URL] }}
+          VAULT_API_URL: ${{ secrets[steps.secretnames.outputs.VAULT_API_URL] }}
+          VAULT_SALT_SECRET: ${{ secrets[steps.secretnames.outputs.VAULT_SALT_SECRET] }}
+          SERVICES_HUB_AUTH_URL: ${{ secrets[steps.secretnames.outputs.SERVICES_HUB_AUTH_URL] }}
+          TXL_HUB_TARGET: ${{ secrets[steps.secretnames.outputs.TXL_HUB_TARGET] }}
+          TXL_HUB_MA: ${{ secrets[steps.secretnames.outputs.TXL_HUB_MA] }}
+          TXL_THREADS_TARGET: ${{ secrets[steps.secretnames.outputs.TXL_THREADS_TARGET] }}
+          TXL_HUB_GATEWAY_URL: ${{ secrets[steps.secretnames.outputs.TXL_HUB_GATEWAY_URL] }}
+          TXL_USER_KEY: ${{ secrets[steps.secretnames.outputs.TXL_USER_KEY] }}
+          TXL_USER_SECRET: ${{ secrets[steps.secretnames.outputs.TXL_USER_SECRET] }}

.goreleaser.yml

@@ -8,8 +8,8 @@ before:
 project_name: space
 builds:
   - id: space
-    env:
-      - CGO_ENABLED=0
+    # env:
+    #   - CGO_ENABLED=0
     ldflags:
       - -s -w -X main.mongousr={{ .Env.MONGO_USR }}
       - -X main.mongopw={{ .Env.MONGO_PW }}
@@ -22,6 +22,7 @@ builds:
       - -X main.textilehub={{ .Env.TXL_HUB_TARGET }}
       - -X main.textilehubma={{ .Env.TXL_HUB_MA }}
       - -X main.textilethreads={{ .Env.TXL_THREADS_TARGET }}
+      - -X main.textilehubgatewayurl={{ .Env.TXL_HUB_GATEWAY_URL }}
       - -X main.textileuserkey={{ .Env.TXL_USER_KEY }}
       - -X main.textileusersecret={{ .Env.TXL_USER_SECRET }}
     main: ./cmd/space-daemon/main.go
@@ -30,8 +31,8 @@ builds:
       - linux
 
   - id: space-darwin
-    env:
-      - CGO_ENABLED=0
+    # env:
+    #   - CGO_ENABLED=0
     ldflags:
       - -s -w -X main.mongousr={{ .Env.MONGO_USR }}
       - -X main.mongopw={{ .Env.MONGO_PW }}
@@ -44,6 +45,7 @@ builds:
       - -X main.textilehub={{ .Env.TXL_HUB_TARGET }}
       - -X main.textilehubma={{ .Env.TXL_HUB_MA }}
       - -X main.textilethreads={{ .Env.TXL_THREADS_TARGET }}
+      - -X main.textilehubgatewayurl={{ .Env.TXL_HUB_GATEWAY_URL }}
       - -X main.textileuserkey={{ .Env.TXL_USER_KEY }}
       - -X main.textileusersecret={{ .Env.TXL_USER_SECRET }}
     main: ./cmd/space-daemon/main.go
@@ -54,8 +56,8 @@ builds:
     #   post: gon -log-level debug ci/gon.hcl
 
   - id: space-win
-    env:
-      - CGO_ENABLED=0
+    # env:
+    #   - CGO_ENABLED=0
     ldflags:
       - -s -w -X main.mongousr={{ .Env.MONGO_USR }}
       - -X main.mongopw={{ .Env.MONGO_PW }}
@@ -68,6 +70,7 @@ builds:
       - -X main.textilehub={{ .Env.TXL_HUB_TARGET }}
       - -X main.textilehubma={{ .Env.TXL_HUB_MA }}
       - -X main.textilethreads={{ .Env.TXL_THREADS_TARGET }}
+      - -X main.textilehubgatewayurl={{ .Env.TXL_HUB_GATEWAY_URL }}
       - -X main.textileuserkey={{ .Env.TXL_USER_KEY }}
       - -X main.textileusersecret={{ .Env.TXL_USER_SECRET }}
     main: ./cmd/space-daemon/main.go

.vscode/launch.json

@@ -10,7 +10,7 @@
       "request": "launch",
       "mode": "auto",
       "program": "${workspaceFolder}/cmd/space-daemon",
-      "env": {},
+      "envFile": "${workspaceFolder}/.env",
       "args": ["-dev=true"]
     }
   ]

Makefile

@@ -13,7 +13,8 @@ build:
 	-X 'main.textilehub=${TXL_HUB_TARGET}' \
 	-X 'main.textilehubma=${TXL_HUB_MA}' \
 	-X 'main.textilethreads=${TXL_THREADS_TARGET}' \
-  -X 'main.textileuserkey=${TXL_USER_KEY}' \
+	-X 'main.textilehubgatewayurl=${TXL_HUB_GATEWAY_URL}' \
+  	-X 'main.textileuserkey=${TXL_USER_KEY}' \
 	-X 'main.textileusersecret=${TXL_USER_SECRET}' \
 	-X 'main.mongohost=${MONGO_HOST}' \
 	-X 'main.mongorepset=${MONGO_REPLICA_SET}'" \

README.md

@@ -186,3 +186,15 @@ server in localhost:6060. See docs how to interact with pprof server here: https
 
 To disable debug mode add this flag to binary arguments
 `-debug=false`
+
+### CI Secrets
+
+Secrets are set by adding them in Github and then specifying them in `release.yml`. Secrets can be constant across environment/stages or be stage specific.
+
+If specified, the release file will dynamically generate the secret name based on the stage by adding a `_DEV` or `_PRD` suffix to the secret name only for the specificed environment variable. It will always use `_PRD` unless the tag ends in `-dev`.  So for example tag `v0.0.15` will use PRD values, while `v0.0.15-dev` will use DEV values.
+
+Stage specific secret names will only be used for secrets `release.yml` that point to the step output instead of the secret name directly (i.e., `SERVICES_API_URL: ${{ secrets[steps.secretnames.outputs.SERVICES_API_URL] }}` instead of `MONGO_REPLICA_SET: ${{ secrets.MONGO_REPLICA_SET }}`.
+
+So to add a new secret:
+* If it's not stage specific then add the secret in GH with no suffix and in `release.yml`, refer to it based on the secret name.
+* If it is stage specific, then create the 2 secrets in GH (ending in `_PRD` and `_DEV`), add the entry in step `secretnames`, and make sure the secret name in the next step points to the step output

app/app.go

@@ -140,15 +140,19 @@ func (a *App) Start(ctx context.Context) error {
 		a.cfg,
 		kc,
 		v,
-		hub.New(appStore, kc, a.cfg),
+		hubAuth,
 		space.WithEnv(a.env),
 	)
 	if svErr != nil {
 		return svErr
 	}
 
 	// setup FUSE FS Handler
-	sfs, err := spacefs.New(fsds.NewSpaceFSDataSource(sv))
+	sfs, err := spacefs.New(fsds.NewSpaceFSDataSource(
+		sv,
+		fsds.WithFilesDataSources(sv),
+		fsds.WithSharedWithMeDataSources(sv),
+	))
 	if err != nil {
 		log.Error("Failed to create space FUSE data source", err)
 		return err

cmd/space-daemon/main.go

@@ -19,27 +19,28 @@ import (
 )
 
 var (
-	cpuprofile        = flag.String("cpuprofile", "", "write cpu profile to `file`")
-	memprofile        = flag.String("memprofile", "", "write memory profile to `file`")
-	debugMode         = flag.Bool("debug", true, "run daemon with debug mode for profiling")
-	devMode           = flag.Bool("dev", false, "run daemon in dev mode to use .env file")
-	ipfsaddr          = flag.String("ipfsaddr", "/ip4/127.0.0.1/tcp/5001", "IPFS multiaddress to connect to (defaults to local node)")
-	ipfsnode          = flag.Bool("ipfsnode", true, "run IPFS embedded into the daemon (defaults to true)")
-	ipfsnodeaddr      string
-	ipfsnodepath      string
-	mongousr          string
-	mongopw           string
-	mongohost         string
-	mongorepset       string
-	spaceapi          string
-	vaultapi          string
-	vaultsaltsecret   string
-	spacehubauth      string
-	textilehub        string
-	textilehubma      string
-	textilethreads    string
-	textileuserkey    string
-	textileusersecret string
+	cpuprofile           = flag.String("cpuprofile", "", "write cpu profile to `file`")
+	memprofile           = flag.String("memprofile", "", "write memory profile to `file`")
+	debugMode            = flag.Bool("debug", true, "run daemon with debug mode for profiling")
+	devMode              = flag.Bool("dev", false, "run daemon in dev mode to use .env file")
+	ipfsaddr             = flag.String("ipfsaddr", "/ip4/127.0.0.1/tcp/5001", "IPFS multiaddress to connect to (defaults to local node)")
+	ipfsnode             = flag.Bool("ipfsnode", true, "run IPFS embedded into the daemon (defaults to true)")
+	ipfsnodeaddr         string
+	ipfsnodepath         string
+	mongousr             string
+	mongopw              string
+	mongohost            string
+	mongorepset          string
+	spaceapi             string
+	vaultapi             string
+	vaultsaltsecret      string
+	spacehubauth         string
+	textilehub           string
+	textilehubma         string
+	textilethreads       string
+	textilehubgatewayurl string
+	textileuserkey       string
+	textileusersecret    string
 )
 
 func main() {
@@ -69,6 +70,7 @@ func main() {
 		TextileHubTarget:     textilehub,
 		TextileHubMa:         textilehubma,
 		TextileThreadsTarget: textilethreads,
+		TextileHubGatewayUrl: textilehubgatewayurl,
 		TextileUserKey:       textileuserkey,
 		TextileUserSecret:    textileusersecret,
 	}

config/config.go

@@ -13,6 +13,7 @@ const (
 	TextileHubTarget         = "space/textileHubTarget"
 	TextileHubMa             = "space/textileHubMa"
 	TextileThreadsTarget     = "space/textileThreadsTarget"
+	TextileHubGatewayUrl     = "space/TextileHubGatewayUrl"
 	TextileUserKey           = "space/textileUserKey"
 	TextileUserSecret        = "space/textileUserSecret"
 	MountFuseDrive           = "space/mountFuseDrive"
@@ -55,6 +56,7 @@ type Flags struct {
 	TextileHubTarget     string
 	TextileHubMa         string
 	TextileThreadsTarget string
+	TextileHubGatewayUrl string
 	TextileUserKey       string
 	TextileUserSecret    string
 }

config/map_config.go

@@ -39,6 +39,7 @@ func NewMap(envVal env.SpaceEnv, flags *Flags) Config {
 		configStr[TextileHubTarget] = os.Getenv(env.TextileHubTarget)
 		configStr[TextileHubMa] = os.Getenv(env.TextileHubMa)
 		configStr[TextileThreadsTarget] = os.Getenv(env.TextileThreadsTarget)
+		configStr[TextileHubGatewayUrl] = os.Getenv(env.TextileHubGatewayUrl)
 		configStr[TextileUserKey] = os.Getenv(env.TextileUserKey)
 		configStr[TextileUserSecret] = os.Getenv(env.TextileUserSecret)
 
@@ -60,11 +61,17 @@ func NewMap(envVal env.SpaceEnv, flags *Flags) Config {
 		configStr[TextileHubTarget] = flags.TextileHubTarget
 		configStr[TextileHubMa] = flags.TextileHubMa
 		configStr[TextileThreadsTarget] = flags.TextileThreadsTarget
+		configStr[TextileHubGatewayUrl] = flags.TextileHubGatewayUrl
 		configStr[TextileUserKey] = flags.TextileUserKey
 		configStr[TextileUserSecret] = flags.TextileUserSecret
 		configBool[Ipfsnode] = flags.Ipfsnode
 	}
 
+	// Temp fix until we move to viper
+	if configStr[Ipfsaddr] == "" {
+		configStr[Ipfsaddr] = "/ip4/127.0.0.1/tcp/5001"
+	}
+
 	c := mapConfig{
 		configStr:  configStr,
 		configInt:  configInt,

core/env/env.go

@@ -24,6 +24,7 @@ const (
 	TextileHubTarget     = "TXL_HUB_TARGET"
 	TextileHubMa         = "TXL_HUB_MA"
 	TextileThreadsTarget = "TXL_THREADS_TARGET"
+	TextileHubGatewayUrl = "TXL_HUB_GATEWAY_URL"
 	TextileUserKey       = "TXL_USER_KEY"
 	TextileUserSecret    = "TXL_USER_SECRET"
 )

core/env/file_env.go

@@ -1,10 +1,11 @@
 package env
 
 import (
-	"github.com/joho/godotenv"
 	syslog "log"
 	"os"
 	"strings"
+
+	"github.com/joho/godotenv"
 )
 
 type spaceEnv struct {
@@ -55,4 +56,4 @@ func (s spaceEnv) LogLevel() string {
 	}
 
 	return ll
-}
+}

core/fsds/config.go

@@ -0,0 +1,57 @@
+package fsds
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/FleekHQ/space-daemon/core/space"
+)
+
+var DefaultBucketName = "personal"
+
+type dataSourceConfig struct {
+	tlfSources []*TLFDataSource
+}
+
+type FSDataSourceConfig func(config *dataSourceConfig)
+
+func WithTLFDataSource(source *TLFDataSource) FSDataSourceConfig {
+	return func(config *dataSourceConfig) {
+		config.tlfSources = append(config.tlfSources, source)
+	}
+}
+
+// Configure the default 'Files` data source to be included as a data source
+func WithFilesDataSources(service space.Service) FSDataSourceConfig {
+	basePath := fmt.Sprintf("%cFiles", os.PathSeparator)
+	return WithTLFDataSource(&TLFDataSource{
+		name:         "Files",
+		basePath:     basePath,
+		FSDataSource: &filesDataSource{service: service},
+	})
+}
+
+// Configure the default 'Shared With Me` data source to be included as a data source
+func WithSharedWithMeDataSources(service space.Service) FSDataSourceConfig {
+	basePath := fmt.Sprintf("%cShared With Me", os.PathSeparator)
+	return WithTLFDataSource(&TLFDataSource{
+		name:     "Shared With Me",
+		basePath: basePath,
+		FSDataSource: &sharedWithMeDataSource{
+			service:     service,
+			maxDirLimit: 1000,
+			cache:       make(map[string]*sharedFileEntry),
+		},
+	})
+}
+
+var blackListedDirEntryNames = map[string]bool{
+	// OSX specific special directories
+	".Trashes":              true,
+	".localized":            true,
+	".fseventsd":            true,
+	".ql_disablethumbnails": true,
+	".ql_disablecache":      true,
+	// special space empty directory file
+	".keep": true,
+}