diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index 02c2b48c7..b57a4b134 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -31,6 +31,7 @@ body: label: Version description: What version are you running? options: + - v0.21.0 - v0.20.6 - v0.20.5 - v0.20.4 diff --git a/README.md b/README.md index 840ceb301..6ddb8b670 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@

- + diff --git a/compose/docker-compose.netclient.yml b/compose/docker-compose.netclient.yml index 9144b76cd..8c184fffb 100644 --- a/compose/docker-compose.netclient.yml +++ b/compose/docker-compose.netclient.yml @@ -3,7 +3,7 @@ version: "3.4" services: netclient: container_name: netclient - image: 'gravitl/netclient:v0.20.6' + image: 'gravitl/netclient:v0.21.0' hostname: netmaker-1 network_mode: host restart: on-failure diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index d17275291..7fbf71136 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -62,7 +62,7 @@ services: coredns: container_name: coredns - image: coredns/coredns + image: coredns/coredns:1.10.1 command: -conf /root/dnsconfig/Corefile env_file: ./netmaker.env depends_on: diff --git a/controllers/dns_test.go b/controllers/dns_test.go index e070cfed4..7d2892db0 100644 --- a/controllers/dns_test.go +++ b/controllers/dns_test.go @@ -400,6 +400,19 @@ func TestValidateDNSCreate(t *testing.T) { assert.NotNil(t, err) assert.Contains(t, err.Error(), "Field validation for 'Name' failed on the 'name_unique' tag") }) + t.Run("WhiteSpace", func(t *testing.T) { + entry := models.DNSEntry{Address: "10.10.10.5", Name: "white space", Network: "skynet"} + err := logic.ValidateDNSCreate(entry) + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "Field validation for 'Name' failed on the 'whitespace' tag") + }) + t.Run("AllSpaces", func(t *testing.T) { + entry := models.DNSEntry{Address: "10.10.10.5", Name: " ", Network: "skynet"} + err := logic.ValidateDNSCreate(entry) + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "Field validation for 'Name' failed on the 'whitespace' tag") + }) + } func createHost() { diff --git a/controllers/docs.go b/controllers/docs.go index 592922982..2cbe329ea 100644 --- a/controllers/docs.go +++ b/controllers/docs.go @@ -10,7 +10,7 @@ // // Schemes: https // BasePath: / -// Version: 0.20.6 +// Version: 0.21.0 // Host: netmaker.io // // Consumes: diff --git a/controllers/migrate.go b/controllers/migrate.go index badc12734..0d365664c 100644 --- a/controllers/migrate.go +++ b/controllers/migrate.go @@ -2,15 +2,21 @@ package controller import ( "encoding/json" + "fmt" + "net" "net/http" + "time" - "github.com/gravitl/netmaker/auth" + "github.com/google/uuid" "github.com/gravitl/netmaker/database" "github.com/gravitl/netmaker/logger" "github.com/gravitl/netmaker/logic" "github.com/gravitl/netmaker/models" + "github.com/gravitl/netmaker/mq" "github.com/gravitl/netmaker/servercfg" "golang.org/x/crypto/bcrypt" + "golang.org/x/exp/slog" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" ) // swagger:route PUT /api/v1/nodes/migrate nodes migrateNode @@ -26,63 +32,182 @@ import ( // 200: nodeJoinResponse func migrate(w http.ResponseWriter, r *http.Request) { data := models.MigrationData{} + host := models.Host{} + node := models.Node{} + nodes := []models.Node{} + server := models.ServerConfig{} err := json.NewDecoder(r.Body).Decode(&data) if err != nil { logger.Log(0, r.Header.Get("user"), "error decoding request body: ", err.Error()) logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest")) return } - - var networksToAdd = []string{} - for i := range data.LegacyNodes { - legacyNode := data.LegacyNodes[i] - record, err := database.FetchRecord(database.NODES_TABLE_NAME, legacyNode.ID) + for i, legacy := range data.LegacyNodes { + record, err := database.FetchRecord(database.NODES_TABLE_NAME, legacy.ID) if err != nil { - logger.Log(0, "no record for legacy node", legacyNode.ID, err.Error()) - continue - } else { - var oldLegacyNode models.LegacyNode - if err = json.Unmarshal([]byte(record), &oldLegacyNode); err != nil { - logger.Log(0, "error decoding legacy node", err.Error()) + slog.Error("legacy node not found", "error", err) + logic.ReturnErrorResponse(w, r, logic.FormatError(fmt.Errorf("legacy node not found %w", err), "badrequest")) + return + } + var legacyNode models.LegacyNode + if err = json.Unmarshal([]byte(record), &legacyNode); err != nil { + slog.Error("decoding legacy node", "errror", err) + logic.ReturnErrorResponse(w, r, logic.FormatError(fmt.Errorf("decode legacy node %w", err), "badrequest")) + return + } + if err := bcrypt.CompareHashAndPassword([]byte(legacyNode.Password), []byte(legacy.Password)); err != nil { + slog.Error("legacy node invalid password", "error", err) + logic.ReturnErrorResponse(w, r, logic.FormatError(fmt.Errorf("invalid password %w", err), "unauthorized")) + return + } + if i == 0 { + host, node = convertLegacyHostNode(legacy) + host.Name = data.HostName + host.HostPass = data.Password + host.OS = data.OS + if err := logic.CreateHost(&host); err != nil { + slog.Error("create host", "error", err) logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest")) - continue + return + } + server = servercfg.GetServerInfo() + if servercfg.GetBrokerType() == servercfg.EmqxBrokerType { + server.MQUserName = host.ID.String() } - if err := bcrypt.CompareHashAndPassword([]byte(oldLegacyNode.Password), []byte(legacyNode.Password)); err != nil { - logger.Log(0, "error decoding legacy password", err.Error()) - logic.ReturnErrorResponse(w, r, logic.FormatError(err, "unauthorized")) - continue + key, keyErr := logic.RetrievePublicTrafficKey() + if keyErr != nil { + slog.Error("retrieving traffickey", "error", err) + logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal")) + return } - networksToAdd = append(networksToAdd, oldLegacyNode.Network) - _ = database.DeleteRecord(database.NODES_TABLE_NAME, oldLegacyNode.ID) + server.TrafficKey = key + } else { + node = convertLegacyNode(legacyNode, host.ID) } - } - if len(networksToAdd) == 0 { - logger.Log(0, "no valid networks to migrate for host", data.NewHost.Name) - logic.ReturnErrorResponse(w, r, logic.FormatError(err, "unauthorized")) - return - } - if !logic.HostExists(&data.NewHost) { - logic.CheckHostPorts(&data.NewHost) - if err = logic.CreateHost(&data.NewHost); err != nil { - logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest")) - return + if err := logic.UpsertNode(&node); err != nil { + slog.Error("update node", "error", err) + continue } + host.Nodes = append(host.Nodes, node.ID.String()) + + nodes = append(nodes, node) } - key, keyErr := logic.RetrievePublicTrafficKey() - if keyErr != nil { - logger.Log(0, "error retrieving key:", keyErr.Error()) - logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal")) - return + if err := logic.UpsertHost(&host); err != nil { + slog.Error("save host", "error", err) } - server := servercfg.GetServerInfo() - server.TrafficKey = key - response := models.RegisterResponse{ - ServerConf: server, - RequestedHost: data.NewHost, + go mq.PublishPeerUpdate() + response := models.HostPull{ + Host: host, + Nodes: nodes, + ServerConfig: server, } w.WriteHeader(http.StatusOK) json.NewEncoder(w).Encode(&response) - logger.Log(0, "successfully migrated host", data.NewHost.Name, data.NewHost.ID.String()) - // notify host of changes, peer and node updates - go auth.CheckNetRegAndHostUpdate(networksToAdd, &data.NewHost) + + slog.Info("migrated nodes") + // check for gateways + for _, node := range data.LegacyNodes { + if node.IsEgressGateway == "yes" { + egressGateway := models.EgressGatewayRequest{ + NodeID: node.ID, + Ranges: node.EgressGatewayRanges, + NatEnabled: node.EgressGatewayNatEnabled, + } + if _, err := logic.CreateEgressGateway(egressGateway); err != nil { + logger.Log(0, "error creating egress gateway for node", node.ID, err.Error()) + } + } + if node.IsIngressGateway == "yes" { + ingressGateway := models.IngressRequest{} + ingressNode, err := logic.CreateIngressGateway(node.Network, node.ID, ingressGateway) + if err != nil { + logger.Log(0, "error creating ingress gateway for node", node.ID, err.Error()) + } + runUpdates(&ingressNode, true) + } + } +} + +func convertLegacyHostNode(legacy models.LegacyNode) (models.Host, models.Node) { + //convert host + host := models.Host{} + host.ID = uuid.New() + host.IPForwarding = models.ParseBool(legacy.IPForwarding) + host.AutoUpdate = servercfg.AutoUpdateEnabled() + host.Interface = "netmaker" + host.ListenPort = int(legacy.ListenPort) + host.MTU = int(legacy.MTU) + host.PublicKey, _ = wgtypes.ParseKey(legacy.PublicKey) + host.MacAddress = net.HardwareAddr(legacy.MacAddress) + host.TrafficKeyPublic = legacy.TrafficKeys.Mine + host.Nodes = append([]string{}, legacy.ID) + host.Interfaces = legacy.Interfaces + //host.DefaultInterface = legacy.Defaul + host.EndpointIP = net.ParseIP(legacy.Endpoint) + host.IsDocker = models.ParseBool(legacy.IsDocker) + host.IsK8S = models.ParseBool(legacy.IsK8S) + host.IsStatic = models.ParseBool(legacy.IsStatic) + node := convertLegacyNode(legacy, host.ID) + return host, node +} + +func convertLegacyNode(legacy models.LegacyNode, hostID uuid.UUID) models.Node { + //convert node + node := models.Node{} + node.ID, _ = uuid.Parse(legacy.ID) + node.HostID = hostID + node.Network = legacy.Network + valid4 := true + valid6 := true + _, cidr4, err := net.ParseCIDR(legacy.NetworkSettings.AddressRange) + if err != nil { + valid4 = false + slog.Warn("parsing address range", "error", err) + } else { + node.NetworkRange = *cidr4 + } + _, cidr6, err := net.ParseCIDR(legacy.NetworkSettings.AddressRange6) + if err != nil { + valid6 = false + slog.Warn("parsing address range6", "error", err) + } else { + node.NetworkRange6 = *cidr6 + } + node.Server = servercfg.GetServer() + node.Connected = models.ParseBool(legacy.Connected) + if valid4 { + node.Address = net.IPNet{ + IP: net.ParseIP(legacy.Address), + Mask: cidr4.Mask, + } + } + if valid6 { + node.Address6 = net.IPNet{ + IP: net.ParseIP(legacy.Address6), + Mask: cidr6.Mask, + } + } + node.Action = models.NODE_NOOP + node.LocalAddress = net.IPNet{ + IP: net.ParseIP(legacy.LocalAddress), + } + node.IsEgressGateway = models.ParseBool(legacy.IsEgressGateway) + node.EgressGatewayRanges = legacy.EgressGatewayRanges + node.IsIngressGateway = models.ParseBool(legacy.IsIngressGateway) + node.IsRelayed = false + node.IsRelay = false + node.RelayedNodes = []string{} + node.DNSOn = models.ParseBool(legacy.DNSOn) + node.PersistentKeepalive = time.Duration(int64(time.Second) * int64(legacy.PersistentKeepalive)) + node.LastModified = time.Now() + node.ExpirationDateTime = time.Unix(legacy.ExpirationDateTime, 0) + node.EgressGatewayNatEnabled = models.ParseBool(legacy.EgressGatewayNatEnabled) + node.EgressGatewayRequest = legacy.EgressGatewayRequest + node.IngressGatewayRange = legacy.IngressGatewayRange + node.IngressGatewayRange6 = legacy.IngressGatewayRange6 + node.DefaultACL = legacy.DefaultACL + node.OwnerID = legacy.OwnerID + node.FailoverNode, _ = uuid.Parse(legacy.FailoverNode) + node.Failover = models.ParseBool(legacy.Failover) + return node } diff --git a/ee/types.go b/ee/types.go index 355bc6d3a..1bae0bb10 100644 --- a/ee/types.go +++ b/ee/types.go @@ -26,13 +26,13 @@ var errValidation = fmt.Errorf(license_validation_err_msg) type LicenseKey struct { LicenseValue string `json:"license_value"` // actual (public) key and the unique value for the key Expiration int64 `json:"expiration"` - UsageServers int `json:"usage_servers"` - UsageUsers int `json:"usage_users"` - UsageClients int `json:"usage_clients"` - UsageHosts int `json:"usage_hosts"` - UsageNetworks int `json:"usage_networks"` - UsageIngresses int `json:"usage_ingresses"` - UsageEgresses int `json:"usage_egresses"` + UsageServers int `json:"limit_servers"` + UsageUsers int `json:"limit_users"` + UsageClients int `json:"limit_clients"` + UsageHosts int `json:"limit_hosts"` + UsageNetworks int `json:"limit_networks"` + UsageIngresses int `json:"limit_ingresses"` + UsageEgresses int `json:"limit_egresses"` Metadata string `json:"metadata"` IsActive bool `json:"is_active"` // yes if active } @@ -46,7 +46,7 @@ type ValidatedLicense struct { // LicenseSecret - the encrypted struct for sending user-id type LicenseSecret struct { AssociatedID string `json:"associated_id" binding:"required"` // UUID for user foreign key to User table - Usage Usage `json:"usage" binding:"required"` + Usage Usage `json:"limits" binding:"required"` } // Usage - struct for license usage diff --git a/go.mod b/go.mod index f73dba38c..235fb9cc8 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,9 @@ go 1.19 require ( github.com/eclipse/paho.mqtt.golang v1.4.3 - github.com/go-playground/validator/v10 v10.15.0 + github.com/go-playground/validator/v10 v10.15.1 github.com/golang-jwt/jwt/v4 v4.5.0 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.3.1 github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 github.com/lib/pq v1.10.9 diff --git a/go.sum b/go.sum index 537e31453..bdcc0c8db 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,8 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.15.0 h1:nDU5XeOKtB3GEa+uB7GNYwhVKsgjAR7VgKoNB6ryXfw= -github.com/go-playground/validator/v10 v10.15.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/go-playground/validator/v10 v10.15.1 h1:BSe8uhN+xQ4r5guV/ywQI4gO59C2raYcGffYWZEjZzM= +github.com/go-playground/validator/v10 v10.15.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -43,8 +43,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= diff --git a/k8s/client/netclient-daemonset.yaml b/k8s/client/netclient-daemonset.yaml index 635e25769..12c897564 100644 --- a/k8s/client/netclient-daemonset.yaml +++ b/k8s/client/netclient-daemonset.yaml @@ -16,7 +16,7 @@ spec: hostNetwork: true containers: - name: netclient - image: gravitl/netclient:v0.20.6 + image: gravitl/netclient:v0.21.0 env: - name: TOKEN value: "TOKEN_VALUE" diff --git a/k8s/client/netclient.yaml b/k8s/client/netclient.yaml index 12a6ea116..f329be78a 100644 --- a/k8s/client/netclient.yaml +++ b/k8s/client/netclient.yaml @@ -28,7 +28,7 @@ spec: # - "" containers: - name: netclient - image: gravitl/netclient:v0.20.6 + image: gravitl/netclient:v0.21.0 env: - name: TOKEN value: "TOKEN_VALUE" diff --git a/k8s/server/netmaker-ui.yaml b/k8s/server/netmaker-ui.yaml index 2d7e62ca5..042ea2fdb 100644 --- a/k8s/server/netmaker-ui.yaml +++ b/k8s/server/netmaker-ui.yaml @@ -15,7 +15,7 @@ spec: spec: containers: - name: netmaker-ui - image: gravitl/netmaker-ui:v0.20.6 + image: gravitl/netmaker-ui:v0.21.0 ports: - containerPort: 443 env: diff --git a/logic/dns.go b/logic/dns.go index 0991c8ce7..15c25deff 100644 --- a/logic/dns.go +++ b/logic/dns.go @@ -3,6 +3,7 @@ package logic import ( "encoding/json" "os" + "regexp" "sort" validator "github.com/go-playground/validator/v10" @@ -203,6 +204,11 @@ func ValidateDNSCreate(entry models.DNSEntry) error { v := validator.New() + _ = v.RegisterValidation("whitespace", func(f1 validator.FieldLevel) bool { + match, _ := regexp.MatchString(`\s`, entry.Name) + return !match + }) + _ = v.RegisterValidation("name_unique", func(fl validator.FieldLevel) bool { num, err := GetDNSEntryNum(entry.Name, entry.Network) return err == nil && num == 0 @@ -227,6 +233,11 @@ func ValidateDNSUpdate(change models.DNSEntry, entry models.DNSEntry) error { v := validator.New() + _ = v.RegisterValidation("whitespace", func(f1 validator.FieldLevel) bool { + match, _ := regexp.MatchString(`\s`, entry.Name) + return !match + }) + _ = v.RegisterValidation("name_unique", func(fl validator.FieldLevel) bool { //if name & net not changing name we are good if change.Name == entry.Name && change.Network == entry.Network { diff --git a/main.go b/main.go index 277742614..bd2ea236e 100644 --- a/main.go +++ b/main.go @@ -28,7 +28,7 @@ import ( "golang.org/x/exp/slog" ) -var version = "v0.20.6" +var version = "v0.21.0" // Start DB Connection and start API Request Handler func main() { diff --git a/models/dnsEntry.go b/models/dnsEntry.go index 73373f1a1..11e9dd6b4 100644 --- a/models/dnsEntry.go +++ b/models/dnsEntry.go @@ -42,8 +42,8 @@ type DNSUpdate struct { // DNSEntry - a DNS entry represented as struct type DNSEntry struct { - Address string `json:"address" bson:"address" validate:"ip"` - Address6 string `json:"address6" bson:"address6"` - Name string `json:"name" bson:"name" validate:"required,name_unique,min=1,max=192"` - Network string `json:"network" bson:"network" validate:"network_exists"` + Address string `json:"address" validate:"ip"` + Address6 string `json:"address6"` + Name string `json:"name" validate:"required,name_unique,min=1,max=192,whitespace"` + Network string `json:"network" validate:"network_exists"` } diff --git a/models/migrate.go b/models/migrate.go index 9972c22c1..978354cb4 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -2,6 +2,8 @@ package models // MigrationData struct needed to create new v0.18.0 node from v.0.17.X node type MigrationData struct { - NewHost Host + HostName string + Password string + OS string LegacyNodes []LegacyNode } diff --git a/models/node.go b/models/node.go index ee10f5597..ed778d00e 100644 --- a/models/node.go +++ b/models/node.go @@ -100,7 +100,6 @@ type Node struct { // LegacyNode - legacy struct for node model type LegacyNode struct { ID string `json:"id,omitempty" bson:"id,omitempty" yaml:"id,omitempty" validate:"required,min=5,id_unique"` - HostID string `json:"hostid,omitempty" bson:"id,omitempty" yaml:"hostid,omitempty" validate:"required,min=5,id_unique"` Address string `json:"address" bson:"address" yaml:"address" validate:"omitempty,ipv4"` Address6 string `json:"address6" bson:"address6" yaml:"address6" validate:"omitempty,ipv6"` LocalAddress string `json:"localaddress" bson:"localaddress" yaml:"localaddress" validate:"omitempty"` @@ -109,7 +108,6 @@ type LegacyNode struct { NetworkSettings Network `json:"networksettings" bson:"networksettings" yaml:"networksettings" validate:"-"` ListenPort int32 `json:"listenport" bson:"listenport" yaml:"listenport" validate:"omitempty,numeric,min=1024,max=65535"` LocalListenPort int32 `json:"locallistenport" bson:"locallistenport" yaml:"locallistenport" validate:"numeric,min=0,max=65535"` - ProxyListenPort int32 `json:"proxy_listen_port" bson:"proxy_listen_port" yaml:"proxy_listen_port" validate:"numeric,min=0,max=65535"` PublicKey string `json:"publickey" bson:"publickey" yaml:"publickey" validate:"required,base64"` Endpoint string `json:"endpoint" bson:"endpoint" yaml:"endpoint" validate:"required,ip"` AllowedIPs []string `json:"allowedips" bson:"allowedips" yaml:"allowedips"` @@ -153,8 +151,6 @@ type LegacyNode struct { FirewallInUse string `json:"firewallinuse" bson:"firewallinuse" yaml:"firewallinuse"` InternetGateway string `json:"internetgateway" bson:"internetgateway" yaml:"internetgateway"` Connected string `json:"connected" bson:"connected" yaml:"connected" validate:"checkyesorno"` - PendingDelete bool `json:"pendingdelete" bson:"pendingdelete" yaml:"pendingdelete"` - Proxy bool `json:"proxy" bson:"proxy" yaml:"proxy"` // == PRO == DefaultACL string `json:"defaultacl,omitempty" bson:"defaultacl,omitempty" yaml:"defaultacl,omitempty" validate:"checkyesornoorunset"` OwnerID string `json:"ownerid,omitempty" bson:"ownerid,omitempty" yaml:"ownerid,omitempty"` @@ -527,7 +523,7 @@ func (ln *LegacyNode) ConvertToNewNode() (*Host, *Node) { func (n *Node) Legacy(h *Host, s *ServerConfig, net *Network) *LegacyNode { l := LegacyNode{} l.ID = n.ID.String() - l.HostID = h.ID.String() + //l.HostID = h.ID.String() l.Address = n.Address.String() l.Address6 = n.Address6.String() l.Interfaces = h.Interfaces diff --git a/release.md b/release.md index 82f009f55..1c363132c 100644 --- a/release.md +++ b/release.md @@ -1,22 +1,19 @@ -# Netmaker v0.20.6 +# Netmaker v0.21.0 ## Whats New -- Extclient Acls -- Force delete host along with all the associated nodes +- Sync clients with server state from UI ## What's Fixed -- Deprecated Proxy -- Solved Race condition for multiple nodes joining network at same time -- Node dns toggle -- Simplified Firewall rules for added stability +- Upgrade Process from v0.17.1 to latest version can be now done seamlessly, please refer docs for more information +- Expired nodes clean up is handled correctly now +- Ext client config generation fixed for ipv6 endpoints +- installation process will only generate certs required for required Domains based on CE or EE +- support for ARM machines on install script ## known issues -- Expired nodes are not getting cleaned up - Windows installer does not install WireGuard - netclient-gui will continously display error dialog if netmaker server is offline -- Incorrect metrics against ext clients -- Host ListenPorts set to 0 after migration from 0.17.1 -> 0.20.6 - Mac IPv6 addresses/route issues - Docker client can not re-join after complete deletion - netclient-gui network tab blank after disconnect diff --git a/scripts/nm-upgrade-0-17-1-to-0-19-0.sh b/scripts/nm-upgrade-0-17-1-to-0-19-0.sh index 359457441..7ece935ff 100644 --- a/scripts/nm-upgrade-0-17-1-to-0-19-0.sh +++ b/scripts/nm-upgrade-0-17-1-to-0-19-0.sh @@ -1,6 +1,6 @@ #!/bin/bash -LATEST="v0.20.6" +LATEST="v0.21.0" INSTALL_PATH="/root" trap restore_old_netmaker_instructions diff --git a/scripts/nm-upgrade.sh b/scripts/nm-upgrade.sh new file mode 100755 index 000000000..34a2df998 --- /dev/null +++ b/scripts/nm-upgrade.sh @@ -0,0 +1,574 @@ +#!/bin/bash + +CONFIG_FILE=netmaker.env +# location of nm-quick.sh (usually `/root`) +SCRIPT_DIR=$(dirname "$(realpath "$0")") +CONFIG_PATH="$SCRIPT_DIR/$CONFIG_FILE" +NM_QUICK_VERSION="0.1.0" +LATEST=$(curl -s https://api.github.com/repos/gravitl/netmaker/releases/latest | grep "tag_name" | cut -d : -f 2,3 | tr -d [:space:],\") + +if [ "$(id -u)" -ne 0 ]; then + echo "This script must be run as root" + exit 1 +fi + +unset INSTALL_TYPE +unset NETMAKER_BASE_DOMAIN + +# usage - displays usage instructions +usage() { + echo "nm-upgrade.sh v$NM_QUICK_VERSION" + echo "usage: ./nm-upgrade.sh" + exit 1 +} + +while getopts v flag; do + case "${flag}" in + v) + usage + exit 0 + ;; + *) + usage + exit 0 + ;; + esac +done + +# print_logo - prints the netmaker logo +print_logo() { + cat <<"EOF" +- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + __ __ ______ ______ __ __ ______ __ __ ______ ______ +/\ "-.\ \ /\ ___\ /\__ _\ /\ "-./ \ /\ __ \ /\ \/ / /\ ___\ /\ == \ +\ \ \-. \ \ \ __\ \/_/\ \/ \ \ \-./\ \ \ \ __ \ \ \ _"-. \ \ __\ \ \ __< + \ \_\\"\_\ \ \_____\ \ \_\ \ \_\ \ \_\ \ \_\ \_\ \ \_\ \_\ \ \_____\ \ \_\ \_\ + \/_/ \/_/ \/_____/ \/_/ \/_/ \/_/ \/_/\/_/ \/_/\/_/ \/_____/ \/_/ /_/ + + +- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +EOF +} + +# set_buildinfo - sets the information based on script input for how the installation should be run +set_buildinfo() { + + MASTERKEY=$(grep MASTER_KEY docker-compose.yml | awk '{print $2;}' | tr -d '"') + EMAIL=$(grep email Caddyfile | awk '{print $2;}' | tr -d '"') + BROKER=$(grep SERVER_NAME docker-compose.yml | awk '{print $2;}' | tr -d '"') + PREFIX="broker." + NETMAKER_BASE_DOMAIN=${BROKER/#$PREFIX} + + + echo "-----------------------------------------------------" + echo "Would you like to install Netmaker Community Edition (CE), or Netmaker Enterprise Edition (EE)?" + echo "EE will require you to create an account at https://app.netmaker.io" + echo "-----------------------------------------------------" + select install_option in "Community Edition" "Enterprise Edition"; do + case $REPLY in + 1) + echo "installing Netmaker CE" + INSTALL_TYPE="ce" + break + ;; + 2) + echo "installing Netmaker EE" + INSTALL_TYPE="ee" + break + ;; + *) echo "invalid option $REPLY" ;; + esac + done + + echo "-----------Build Options-----------------------------" + echo " EE or CE: $INSTALL_TYPE" + echo " Version: $LATEST" + echo " Installer: v$NM_QUICK_VERSION" + echo "-----------------------------------------------------" + +} + +# install_yq - install yq if not present +install_yq() { + if ! command -v yq &>/dev/null; then + wget -qO /usr/bin/yq https://github.com/mikefarah/yq/releases/download/v4.31.1/yq_linux_$(dpkg --print-architecture) + chmod +x /usr/bin/yq + fi + set +e + if ! command -v yq &>/dev/null; then + set -e + wget -qO /usr/bin/yq https://github.com/mikefarah/yq/releases/download/v4.31.1/yq_linux_amd64 + chmod +x /usr/bin/yq + fi + set -e + if ! command -v yq &>/dev/null; then + echo "failed to install yq. Please install yq and try again." + echo "https://github.com/mikefarah/yq/#install" + exit 1 + fi +} + +# install and run upgrade tool +upgrade() { + wget -qO /tmp/nm-upgrade https://fileserver.netmaker.io/upgrade/nm-upgrade-${ARCH} + chmod +x /tmp/nm-upgrade + echo "generating netclient configuration files" + /tmp/nm-upgrade +} + +# setup_netclient - installs netclient +setup_netclient() { + wget -qO netclient https://github.com/gravitl/netclient/releases/download/$LATEST/netclient-linux-$ARCH + chmod +x netclient + ./netclient install -v 3 +} + + +# wait_seconds - wait for the specified period of time +wait_seconds() { ( + for ((a = 1; a <= $1; a++)); do + echo ". . ." + sleep 1 + done +); } + +# confirm - get user input to confirm that they want to perform the next step +confirm() { ( + while true; do + read -p 'Does everything look right? [y/n]: ' yn + case $yn in + [Yy]*) + override="true" + break + ;; + [Nn]*) + echo "exiting..." + exit 1 + # TODO start from the beginning instead + ;; + *) echo "Please answer yes or no." ;; + esac + done +) } + +save_config() { ( + echo "Saving the config to $CONFIG_PATH" + touch "$CONFIG_PATH" + save_config_item NM_EMAIL "$EMAIL" + save_config_item NM_DOMAIN "$NETMAKER_BASE_DOMAIN" + save_config_item UI_IMAGE_TAG "$LATEST" + # version-specific entries + if [ "$INSTALL_TYPE" = "ee" ]; then + save_config_item NETMAKER_TENANT_ID "$TENANT_ID" + save_config_item LICENSE_KEY "$LICENSE_KEY" + save_config_item METRICS_EXPORTER "on" + save_config_item PROMETHEUS "on" + save_config_item SERVER_IMAGE_TAG "$LATEST-ee" + else + save_config_item METRICS_EXPORTER "off" + save_config_item PROMETHEUS "off" + save_config_item SERVER_IMAGE_TAG "$LATEST" + fi + # copy entries from the previous config + local toCopy=("SERVER_HOST" "MASTER_KEY" "TURN_USERNAME" "TURN_PASSWORD" "MQ_USERNAME" "MQ_PASSWORD" + "INSTALL_TYPE" "NODE_ID" "DNS_MODE" "NETCLIENT_AUTO_UPDATE" "API_PORT" + "CORS_ALLOWED_ORIGIN" "DISPLAY_KEYS" "DATABASE" "SERVER_BROKER_ENDPOINT" "STUN_PORT" "VERBOSITY" + "TURN_PORT" "USE_TURN" "DEBUG_MODE" "TURN_API_PORT" "REST_BACKEND" + "DISABLE_REMOTE_IP_CHECK" "NETCLIENT_ENDPOINT_DETECTION" "TELEMETRY" "AUTH_PROVIDER" "CLIENT_ID" "CLIENT_SECRET" + "FRONTEND_URL" "AZURE_TENANT" "OIDC_ISSUER" "EXPORTER_API_PORT") + for name in "${toCopy[@]}"; do + save_config_item $name "${!name}" + done + # preserve debug entries + if test -n "$NM_SKIP_BUILD"; then + save_config_item NM_SKIP_BUILD "$NM_SKIP_BUILD" + fi + if test -n "$NM_SKIP_CLONE"; then + save_config_item NM_SKIP_CLONE "$NM_SKIP_CLONE" + fi + if test -n "$NM_SKIP_DEPS"; then + save_config_item NM_SKIP_DEPS "$NM_SKIP_DEPS" + fi +); } + +save_config_item() { ( + local NAME="$1" + local VALUE="$2" + #echo "$NAME=$VALUE" + if test -z "$VALUE"; then + # load the default for empty values + VALUE=$(awk -F'=' "/^$NAME/ { print \$2}" "$SCRIPT_DIR/netmaker.default.env") + # trim quotes for docker + VALUE=$(echo "$VALUE" | sed -E "s|^(['\"])(.*)\1$|\2|g") + #echo "Default for $NAME=$VALUE" + fi + # TODO single quote passwords + if grep -q "^$NAME=" "$CONFIG_PATH"; then + # TODO escape | in the value + sed -i "s|$NAME=.*|$NAME=$VALUE|" "$CONFIG_PATH" + else + echo "$NAME=$VALUE" >>"$CONFIG_PATH" + fi +); } + +# install_dependencies - install necessary packages to run netmaker +install_dependencies() { + + if test -n "$NM_SKIP_DEPS"; then + return + fi + + echo "checking dependencies..." + + OS=$(uname) + if [ -f /etc/debian_version ]; then + dependencies="git wireguard wireguard-tools dnsutils jq docker.io docker-compose grep gawk" + update_cmd='apt update' + install_cmd='apt-get install -y' + elif [ -f /etc/alpine-release ]; then + dependencies="git wireguard jq docker.io docker-compose grep gawk" + update_cmd='apk update' + install_cmd='apk --update add' + elif [ -f /etc/centos-release ]; then + dependencies="git wireguard jq bind-utils docker.io docker-compose grep gawk" + update_cmd='yum update' + install_cmd='yum install -y' + elif [ -f /etc/fedora-release ]; then + dependencies="git wireguard bind-utils jq docker.io docker-compose grep gawk" + update_cmd='dnf update' + install_cmd='dnf install -y' + elif [ -f /etc/redhat-release ]; then + dependencies="git wireguard jq docker.io bind-utils docker-compose grep gawk" + update_cmd='yum update' + install_cmd='yum install -y' + elif [ -f /etc/arch-release ]; then + dependencies="git wireguard-tools dnsutils jq docker.io docker-compose grep gawk" + update_cmd='pacman -Sy' + install_cmd='pacman -S --noconfirm' + elif [ "${OS}" = "FreeBSD" ]; then + dependencies="git wireguard wget jq docker.io docker-compose grep gawk" + update_cmd='pkg update' + install_cmd='pkg install -y' + else + install_cmd='' + fi + + if [ -z "${install_cmd}" ]; then + echo "OS unsupported for automatic dependency install" + # TODO shouldnt exit, check if deps available, if not + # ask the user to install manually and continue when ready + exit 1 + fi + # TODO add other supported architectures + ARCH=$(uname -m) + if [ "$ARCH" = "x86_64" ]; then + ARCH=amd64 + elif [ "$ARCH" = "aarch64" ]; then + ARCH=arm64 + else + echo "Unsupported architechure" + # exit 1 + fi + set -- $dependencies + + ${update_cmd} + + while [ -n "$1" ]; do + if [ "${OS}" = "FreeBSD" ]; then + is_installed=$(pkg check -d $1 | grep "Checking" | grep "done") + if [ "$is_installed" != "" ]; then + echo " " $1 is installed + else + echo " " $1 is not installed. Attempting install. + ${install_cmd} $1 + sleep 5 + is_installed=$(pkg check -d $1 | grep "Checking" | grep "done") + if [ "$is_installed" != "" ]; then + echo " " $1 is installed + elif [ -x "$(command -v $1)" ]; then + echo " " $1 is installed + else + echo " " FAILED TO INSTALL $1 + echo " " This may break functionality. + fi + fi + else + if [ "${OS}" = "OpenWRT" ] || [ "${OS}" = "TurrisOS" ]; then + is_installed=$(opkg list-installed $1 | grep $1) + else + is_installed=$(dpkg-query -W --showformat='${Status}\n' $1 | grep "install ok installed") + fi + if [ "${is_installed}" != "" ]; then + echo " " $1 is installed + else + echo " " $1 is not installed. Attempting install. + ${install_cmd} $1 + sleep 5 + if [ "${OS}" = "OpenWRT" ] || [ "${OS}" = "TurrisOS" ]; then + is_installed=$(opkg list-installed $1 | grep $1) + else + is_installed=$(dpkg-query -W --showformat='${Status}\n' $1 | grep "install ok installed") + fi + if [ "${is_installed}" != "" ]; then + echo " " $1 is installed + elif [ -x "$(command -v $1)" ]; then + echo " " $1 is installed + else + echo " " FAILED TO INSTALL $1 + echo " " This may break functionality. + fi + fi + fi + shift + done + + echo "-----------------------------------------------------" + echo "dependency check complete" + echo "-----------------------------------------------------" +} + +# set_install_vars - sets the variables that will be used throughout installation +set_install_vars() { + + IP_ADDR=$(dig -4 myip.opendns.com @resolver1.opendns.com +short) + if [ "$IP_ADDR" = "" ]; then + IP_ADDR=$(curl -s ifconfig.me) + fi + if [ "$NETMAKER_BASE_DOMAIN" = "" ]; then + NETMAKER_BASE_DOMAIN=nm.$(echo $IP_ADDR | tr . -).nip.io + fi + SERVER_HOST=$IP_ADDR + if test -z "$MASTER_KEY"; then + MASTER_KEY=$( + tr -dc A-Za-z0-9 &1) + + if [[ "$i" == 8 ]]; then + echo " Caddy is having an issue setting up certificates, please investigate (docker logs caddy)" + echo " Exiting..." + exit 1 + elif [[ "$curlresponse" == *"failed to verify the legitimacy of the server"* ]]; then + echo " Certificates not yet configured, retrying..." + + elif [[ "$curlresponse" == *"left intact"* ]]; then + echo " Certificates ok" + break + else + secs=$(($i * 5 + 10)) + echo " Issue establishing connection...retrying in $secs seconds..." + fi + sleep $secs + done + +} + +# print_success - prints a success message upon completion +print_success() { + echo "-----------------------------------------------------------------" + echo "-----------------------------------------------------------------" + echo "Netmaker setup is now complete. You are ready to begin using Netmaker." + echo "Visit dashboard.$NETMAKER_BASE_DOMAIN to log in" + echo "-----------------------------------------------------------------" + echo "-----------------------------------------------------------------" +} + +cleanup() { + echo "Stopping all containers..." + local containers=("mq" "netmaker-ui" "coredns" "turn" "caddy" "netmaker" "netmaker-exporter" "prometheus" "grafana") + for name in "${containers[@]}"; do + local running=$(docker ps | grep -w "$name") + local exists=$(docker ps -a | grep -w "$name") + if test -n "$running"; then + docker stop "$name" 1>/dev/null + fi + if test -n "$exists"; then + docker rm "$name" 1>/dev/null + fi + done +} + +# print netmaker logo +print_logo + +# read the config +if [ -f "$CONFIG_PATH" ]; then + echo "Using config: $CONFIG_PATH" + source "$CONFIG_PATH" + if [ "$UPGRADE_FLAG" = "yes" ]; then + INSTALL_TYPE="ee" + fi +fi + +# setup the build instructions +set_buildinfo + +set +e + +# install necessary packages +install_dependencies + +# install yq if necessary +install_yq + +set -e + +# get user input for variables +set_install_vars + +set +e +cleanup +set -e + +# get upgrade tool and run +upgrade + +# get and set config files, startup docker-compose +install_netmaker + +set +e + +# make sure Caddy certs are working +test_connection + +set -e + +# install netclient +setup_netclient + + +# print success message +print_success + diff --git a/swagger.yaml b/swagger.yaml index c71d63877..77c7de77b 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -704,7 +704,7 @@ info: API calls must be authenticated via a header of the format -H “Authorization: Bearer ” There are two methods to obtain YOUR_SECRET_KEY: 1. Using the masterkey. By default, this value is “secret key,” but you should change this on your instance and keep it secure. This value can be set via env var at startup or in a config file (config/environments/< env >.yaml). See the [Netmaker](https://docs.netmaker.org/index.html) documentation for more details. 2. Using a JWT received for a node. This can be retrieved by calling the /api/nodes//authenticate endpoint, as documented below. title: Netmaker - version: 0.20.6 + version: 0.21.0 paths: /api/dns: get: