Dev (#6)

* fix etcd scale up and down

* enable tls server

* update precommit

* remove ssl dir from dockerfile

* save framework and framework config with frameworkName as prefix.

* fix redis prefix

* fix m3s status and scale feature.

* add makefile for easy building

* add and improve version information

* save config after restart

* check if the ip port is already in use

* fix: search used ports

* save framework_config during heartbeat.

* update bootstrap server

* add etcdconstraint and statetime for future use

* fix random ports

* change revive and channel handling

* Renamed ETCD_CONSTRAINT to K3S_ETCD_CONSTRAINT (#5)

for consistency

* change random port to dynamic

* add version as flag

* add mesos-m3s bin into gitignore

* make url to the version files customabel

* change api urls

* boostrap update check version

* change k3s installation method

* replace update and boostrap parameters in via makefile

* change k3s installation method

* change k3s installation method

* add docker network support

* Seperate K3S Server and Agent Memory and CPU resource definition

* change docker parameter

* update version file format

* change bootstrap body

* add version info into bootstrap

* update version info

* update version info

* fix docker parameter error

* add force image pull, fix docker network default value

* debug: enable docker instead of crictl

* change k3s to use docker engine.

* add cgroup parser in bootstrap

* fix cgroup parser in bootstrap

* add DOCKER_SHM_SIZE variable to configure shm-size

* add docker flag to choose container engine, change k3s server string

* add nix-shell

* change default bootstrap file url

* UPDATE: changelog version to 0.3.0

Co-authored-by: Marcel Neuhausler <[email protected]>

.gitignore

@@ -9,3 +9,6 @@ bootstrap/*.db
 bootstrap/*.db-shm
 bootstrap/*.db-wal
 mesos_cli/m3s/__pycache__
+ssl/
+vendor/
+mesos-m3s

.golangci.yaml

@@ -0,0 +1,113 @@
+linters-settings:
+  depguard:
+    list-type: blacklist
+    packages:
+      # logging is allowed only by logutils.Log, logrus
+      # is allowed to use only in logutils package
+  dupl:
+    threshold: 100
+  funlen:
+    lines: 100
+    statements: 50
+  gci:
+    local-prefixes: github.com/golangci/golangci-lint
+  goconst:
+    min-len: 2
+    min-occurrences: 2
+  gocyclo:
+    min-complexity: 15
+  goimports:
+    local-prefixes: github.com/golangci/golangci-lint
+  gomnd:
+    settings:
+      mnd:
+        # don't include the "operation" and "assign"
+        checks: argument,case,condition,return
+        ignored-numbers: 0,1,2,3
+        ignored-functions: strings.SplitN
+
+  govet:
+    check-shadowing: true
+    settings:
+      printf:
+        funcs:
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf
+  lll:
+    line-length: 140
+  misspell:
+    locale: US
+  nolintlint:
+    allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space)
+    allow-unused: false # report any unused nolint directives
+    require-explanation: false # don't require an explanation for nolint directives
+    require-specific: false # don't require nolint directives to be specific about which linter is being skipped
+
+linters:
+  disable-all: true
+  enable:
+    - bodyclose
+    - deadcode
+    - depguard
+    - dogsled
+    - exportloopref
+    - gocyclo
+    - gofmt
+    - goimports
+    - goprintffuncname
+    - gosec
+    - gosimple
+    - govet
+    - ineffassign
+    - misspell
+    - nakedret
+    - nolintlint
+    - rowserrcheck
+    - staticcheck
+    - structcheck
+    - typecheck
+    - unconvert
+    - unparam
+    - unused
+    - varcheck
+    - whitespace
+
+  # don't enable:
+  # - asciicheck
+  # - scopelint
+  # - gochecknoglobals
+  # - gocognit
+  # - godot
+  # - godox
+  # - goerr113
+  # - interfacer
+  # - maligned
+  # - nestif
+  # - prealloc
+  # - testpackage
+  # - revive
+  # - wsl
+
+issues:
+  # Excluding configuration per-path, per-linter, per-text and per-source
+  exclude-rules:
+    - path: _test\.go
+      linters:
+        - gomnd
+
+    - path: pkg/golinters/errcheck.go
+      text: "SA1019: errCfg.Exclude is deprecated: use ExcludeFunctions instead"
+    - path: pkg/commands/run.go
+      text: "SA1019: lsc.Errcheck.Exclude is deprecated: use ExcludeFunctions instead"
+    - path: pkg/commands/run.go
+      text: "SA1019: e.cfg.Run.Deadline is deprecated: Deadline exists for historical compatibility and should not be used."
+
+run:
+  timeout: 5m
+  skip-dirs:
+    - test/testdata_etc
+    - internal/cache
+    - internal/renameio
+    - internal/robustio

.pre-commit-config.yaml

@@ -8,13 +8,14 @@ repos:
     exclude: ^vendor/
   - id: trailing-whitespace
     exclude: ^vendor/
+  - id: detect-private-key
 
 - repo: https://github.com/ambv/black
   rev: 21.7b0
   hooks:
   - id: black
     exclude: ^vendor/
-    language_version: python3.9
+    language_version: python3.10
 
 - repo: git://github.com/dnephin/pre-commit-golang
   rev: v0.4.0
@@ -25,3 +26,6 @@ repos:
       exclude: ^vendor/
     - id: go-imports
       exclude: ^vendor/
+    - id: golangci-lint
+      args: [--verbose]
+      exclude: ^vendor/

.version.json

@@ -1,5 +1 @@
-[{
-  "bootstrap_build":"20211104%.H2611",
-  "m3s_build":"20211016",
-  "m3s_version":"v0.2.0"
-}]
+{"m3sVersion": {	"gitVersion": "v0.2.0-87-gd859f76",	"buildDate": "2022-03-05T15:41:02Z"}}

Dockerfile

@@ -7,15 +7,17 @@ COPY . /build/
 RUN apk add git && \
     go get -d
 
-RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags "-X main.MinVersion=`date -u +%Y%m%d%.H%M%S` -extldflags \"-static\"" -o main .
+ARG TAG
+ARG BUILDDATE
+ARG VERSION_URL
+RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags "-X main.BuildVersion=$BUILDDATE -X main.GitVersion=$TAG -X main.VersionURL=$VERSION_URL -extldflags \"-static\"" -o main .
 
 
 FROM alpine
 LABEL maintainer="Andreas Peters <[email protected]>"
 
 RUN apk add --no-cache ca-certificates
 RUN adduser -S -D -H -h /app appuser
-
 USER appuser
 
 COPY --from=builder /build/main /app/

Makefile

@@ -0,0 +1,61 @@
+#Dockerfile vars
+
+#vars
+IMAGENAME=mesos-m3s
+REPO=localhost:5000
+TAG=`git describe`
+BUILDDATE=`date -u +%Y-%m-%dT%H:%M:%SZ`
+IMAGEFULLNAME=${REPO}/${IMAGENAME}
+IMAGEFULLNAMEPUB=avhost/${IMAGENAME}
+BRANCH=`git symbolic-ref --short HEAD`
+VERSION_URL=https://raw.githubusercontent.com/AVENTER-UG/mesos-m3s/${BRANCH}/.version.json
+
+.PHONY: help build bootstrap all docs publish push version
+
+help:
+	    @echo "Makefile arguments:"
+	    @echo ""
+	    @echo "Makefile commands:"
+			@echo "push"
+	    @echo "build"
+			@echo "build-bin"
+	    @echo "all"
+			@echo "docs"
+			@echo "publish"
+			@echo "version"
+			@echo ${TAG}
+
+.DEFAULT_GOAL := all
+
+build:
+	@echo ">>>> Build docker image"
+	@docker buildx build --build-arg TAG=${TAG} --build-arg BUILDDATE=${BUILDDATE} --build-arg VERSION_URL=${VERSION_URL} -t ${IMAGEFULLNAME}:${BRANCH} .
+
+push:
+	@echo ">>>> Push into private repo"
+	@docker push localhost:5000/mesos-m3s:dev
+
+build-bin:
+	@echo ">>>> Build binary"
+	@CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags "-X main.BuildVersion=${BUILDDATE} -X main.GitVersion=${TAG} -X main.VersionURL=${VERSION_URL} -extldflags \"-static\"" .
+
+bootstrap:
+	@echo ">>>> Build bootstrap"
+	$(MAKE) -C $@
+
+publish:
+	@echo ">>>> Publish docker image"
+	@docker tag ${IMAGEFULLNAME}:${BRANCH} ${IMAGEFULLNAMEPUB}:${BRANCH}
+	@docker push ${IMAGEFULLNAMEPUB}:${BRANCH}
+
+docs:
+	@echo ">>>> Build docs"
+	$(MAKE) -C $@
+
+version:
+	@echo ">>>> Generate version file"
+	@echo "{\"m3sVersion\": {	\"gitVersion\": \"${TAG}\",	\"buildDate\": \"${BUILDDATE}\"}}" > .version.json
+	@cat .version.json
+	@echo "Saved under .version.json"
+
+all: bootstrap build version

api/V0GetCountK3SAgent.go

@@ -21,7 +21,7 @@ func V0GetCountK3SAgent(w http.ResponseWriter, r *http.Request) {
 	var count cfg.Count
 
 	count.Scale = config.K3SAgentMax
-	count.Running = CountRedisKey(config.PrefixHostname + "agent:*")
+	count.Running = CountRedisKey(framework.FrameworkName + ":agent:*")
 
 	d, err := json.Marshal(count)
 

api/V0GetKubeVersion.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"encoding/json"
 	"io/ioutil"
 	"net/http"
 	"strconv"
@@ -21,7 +22,7 @@ func V0GetKubeVersion(w http.ResponseWriter, r *http.Request) {
 	}
 
 	client := &http.Client{}
-	req, _ := http.NewRequest("GET", "http://"+config.M3SBootstrapServerHostname+":"+strconv.Itoa(config.M3SBootstrapServerPort)+"/api/k3s/v0/version", nil)
+	req, _ := http.NewRequest("GET", "http://"+config.M3SBootstrapServerHostname+":"+strconv.Itoa(config.M3SBootstrapServerPort)+"/api/m3s/bootstrap/v0/version", nil)
 	req.Close = true
 	res, err := client.Do(req)
 
@@ -44,7 +45,19 @@ func V0GetKubeVersion(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	err = json.Unmarshal(content, &config.Version)
+	if err != nil {
+		logrus.Error("V0GetKubeVersion: Error 3 ", err)
+		return
+	}
+
+	d, err := json.Marshal(&config.Version)
+	if err != nil {
+		logrus.Error("V0GetKubeVersion: Error 4 ", err)
+		return
+	}
+
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
 	w.Header().Set("Api-Service", "v0")
-	w.Write([]byte(content))
+	w.Write(d)
 }

api/V0GetKubeconfig.go

@@ -22,7 +22,7 @@ func V0GetKubeconfig(w http.ResponseWriter, r *http.Request) {
 	}
 
 	client := &http.Client{}
-	req, _ := http.NewRequest("GET", "http://"+config.M3SBootstrapServerHostname+":"+strconv.Itoa(config.M3SBootstrapServerPort)+"/api/k3s/v0/config", nil)
+	req, _ := http.NewRequest("GET", "http://"+config.M3SBootstrapServerHostname+":"+strconv.Itoa(config.M3SBootstrapServerPort)+"/api/m3s/bootstrap/v0/config", nil)
 	req.Close = true
 	res, err := client.Do(req)
 
@@ -52,5 +52,4 @@ func V0GetKubeconfig(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
 	w.Header().Set("Api-Service", "v0")
 	w.Write([]byte(kubconf))
-
 }

api/V0ScaleAgent.go

@@ -31,14 +31,12 @@ func V0ScaleK3SAgent(w http.ResponseWriter, r *http.Request) {
 
 		d = []byte(strconv.Itoa(newCount - oldCount))
 
-		err := SaveConfig()
-		if err != nil {
-			d = ErrorMessage(1, "V0ScaleK3SAgent", "Could not save config data")
-		}
+		// Save current config
+		SaveConfig()
 
 		// if scale down, kill not needes agents
 		if newCount < oldCount {
-			keys := GetAllRedisKeys(config.PrefixHostname + "agent:*")
+			keys := GetAllRedisKeys(framework.FrameworkName + ":agent:*")
 
 			for keys.Next(config.RedisCTX) {
 				if newCount < oldCount {

api/V0ScaleEtcd.go

@@ -30,13 +30,12 @@ func V0ScaleEtcd(w http.ResponseWriter, r *http.Request) {
 
 		d = []byte(strconv.Itoa(newCount - oldCount))
 
-		err := SaveConfig()
-		if err != nil {
-			d = ErrorMessage(1, "V0ScaleEtcd", "Could not save config data")
-		}
+		// Save current config
+		SaveConfig()
+
 		// if scale down, kill not needes agents
 		if newCount < oldCount {
-			keys := GetAllRedisKeys(config.PrefixHostname + "agent:*")
+			keys := GetAllRedisKeys(framework.FrameworkName + ":etcd:*")
 
 			for keys.Next(config.RedisCTX) {
 				if newCount < oldCount {

api/V0StatusK8s.go

@@ -22,7 +22,7 @@ func V0StatusK8s(w http.ResponseWriter, r *http.Request) {
 	}
 
 	client := &http.Client{}
-	req, _ := http.NewRequest("GET", "http://"+config.M3SBootstrapServerHostname+":"+strconv.Itoa(config.M3SBootstrapServerPort)+"/status?verbose", nil)
+	req, _ := http.NewRequest("GET", "http://"+config.M3SBootstrapServerHostname+":"+strconv.Itoa(config.M3SBootstrapServerPort)+"/api/m3s/bootstrap/v0/status?verbose", nil)
 	req.Close = true
 	res, err := client.Do(req)
 
@@ -47,5 +47,5 @@ func V0StatusK8s(w http.ResponseWriter, r *http.Request) {
 
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
 	w.Header().Set("Api-Service", "v0")
-	w.Write([]byte(content))
+	w.Write(content)
 }

api/V0StatusM3s.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"net/http"
 
+	mesosutil "github.com/AVENTER-UG/mesos-util"
 	"github.com/gorilla/mux"
 	"github.com/sirupsen/logrus"
 )
@@ -19,6 +20,7 @@ func V0StatusM3s(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	getStatus()
 	status := config.M3SStatus
 
 	d, _ := json.Marshal(&status)
@@ -30,3 +32,28 @@ func V0StatusM3s(w http.ResponseWriter, r *http.Request) {
 
 	w.Write(d)
 }
+
+func getStatus() {
+	config.M3SStatus.Etcd = map[string]string{}
+	config.M3SStatus.Agent = map[string]string{}
+	config.M3SStatus.Server = map[string]string{}
+	services := []string{"etcd", "server", "agent"}
+
+	for _, service := range services {
+		keys := GetAllRedisKeys(framework.FrameworkName + ":" + service + ":*")
+
+		for keys.Next(config.RedisCTX) {
+			key := GetRedisKey(keys.Val())
+			var task mesosutil.Command
+			json.Unmarshal([]byte(key), &task)
+
+			if service == "etcd" {
+				config.M3SStatus.Etcd[task.TaskID] = task.State
+			} else if service == "agent" {
+				config.M3SStatus.Agent[task.TaskID] = task.State
+			} else if service == "server" {
+				config.M3SStatus.Server[task.TaskID] = task.State
+			}
+		}
+	}
+}