From 5c38b5b2d0e6cb49d16fdd09f2a139ba5ee2324f Mon Sep 17 00:00:00 2001 From: Matthew R Kasun Date: Mon, 14 Aug 2023 13:44:10 -0400 Subject: [PATCH 01/11] migration (#2509) * create gateways during migration * set version for testing * restruct migration * debug logging * enforce unique names for ext client names (#2476) * enforce unique names for ext client names * only check for unique id on creation * check for unique id if changed * prune(NET-483): remove defunct host.internetgateway field (#2487) * don't reference host on err (#2493) * deprecrate netclient install scripts (#2490) * Net 500: validate network parameter passed to node endpoints (#2480) * enforce unique names for ext client names * only check for unique id on creation * check for unique id if changed * validate network parameter passed to node endpoints --------- Co-authored-by: Abhishek K <32607604+abhishek9686@users.noreply.github.com> * NET-513 (#2492) nm-certs.sh now requests certificate for EE and CE edition domains accordingly. * [NET-404] Run in limited mode when ee checks fail (#2474) * Add limited http handlers functionality to rest handler * Export ee.errValidation (ee.ErrValidation) * Export a fatal error handled by the hook manager * Export a new status variable for unlicensed server * Mark server as unlicensed when ee checks fail * Handle license validation failures with a (re)boot in a limited state * Revert "Export a fatal error handled by the hook manager" This reverts commit 069c21974a8d36e889c73ad78023448d787d62a5. * Revert "Export ee.errValidation (ee.ErrValidation)" This reverts commit 59dbab8c79773ca5d879f28cbaf53f3dd4297b9b. * Revert "Add limited http handlers functionality to rest handler" This reverts commit e2f1f28facaca54713db76a588839cd2733cf673. * Revert "Handle license validation failures with a (re)boot in a limited state" This reverts commit 58cfbbaf522a1345aac1fa67964ebff0a6d60cd8. * Revert "Mark server as unlicensed when ee checks fail" This reverts commit 77c6dbdd3c9cfa6e7d6becedef6251e8617ae367. * Handle license validation failures with a middleware * Forbid responses if unlicensed ee and not in status api * Remove unused func * feat(NET-449): add sync feature to request a host pull from server (#2491) * fix(NET-486): change client name length validation (#2498) set limit to 5<=x<=32 * [NET-477] Pick AMB URL dynamically (#2489) * Introduce config for environment * Introduce func to get environment * Choose accounts api host from environment * Test the ee package on workflows * Use build tag ee for license_test.go * [Feature]: nm-quick script tackling arm TODO support (#2488) * domain flag for auto installs * use static servers with custom domain (#2421) * send delete peer update always * fix add/remove host api calls * keep mq updates in a single go func * move branch test logic to devops (#2443) * handle IOT OS * save server name to env (#2460) * ensure branch test servers available after test runs (#2467) * save server name to env * free server always; add PR to discord messages * use correct method to delete droplets (#2468) * quick fix for the launcher * removed exit when triggering not supported exit and removed the TODO comments related to this issue --------- Co-authored-by: Matthew R Kasun Co-authored-by: Alex Feiszli <31018251+afeiszli@users.noreply.github.com> Co-authored-by: Christopher Blaha Co-authored-by: Abhishek Kondur Co-authored-by: Abhishek K <32607604+abhishek9686@users.noreply.github.com> * rebase conflict * include pass and os in mirgration data * node network ranges * remove debugging logs * add gateways * use sent node * upgrade shell script * associate node to host during migration * add node to host.Nodes and publish peer update * save host outside loop * fix script name * simplify upgrade script * don't migrate relays * simplify upgrade script even more * guard against blank address or address6 * typos --------- Co-authored-by: Aceix Co-authored-by: Abhishek K <32607604+abhishek9686@users.noreply.github.com> Co-authored-by: Farukh Khan Co-authored-by: Gabriel de Souza Seibel Co-authored-by: bornav <51048565+bornav@users.noreply.github.com> Co-authored-by: Alex Feiszli <31018251+afeiszli@users.noreply.github.com> Co-authored-by: Christopher Blaha Co-authored-by: Abhishek Kondur --- controllers/migrate.go | 212 ++++++++++++--- models/migrate.go | 4 +- models/node.go | 6 +- scripts/nm-upgrade.sh | 574 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 747 insertions(+), 49 deletions(-) create mode 100755 scripts/nm-upgrade.sh diff --git a/controllers/migrate.go b/controllers/migrate.go index badc12734..fa03fccf9 100644 --- a/controllers/migrate.go +++ b/controllers/migrate.go @@ -2,15 +2,22 @@ package controller import ( "encoding/json" + "fmt" + "net" "net/http" + "strconv" + "time" - "github.com/gravitl/netmaker/auth" + "github.com/google/uuid" "github.com/gravitl/netmaker/database" "github.com/gravitl/netmaker/logger" "github.com/gravitl/netmaker/logic" "github.com/gravitl/netmaker/models" + "github.com/gravitl/netmaker/mq" "github.com/gravitl/netmaker/servercfg" "golang.org/x/crypto/bcrypt" + "golang.org/x/exp/slog" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" ) // swagger:route PUT /api/v1/nodes/migrate nodes migrateNode @@ -26,63 +33,182 @@ import ( // 200: nodeJoinResponse func migrate(w http.ResponseWriter, r *http.Request) { data := models.MigrationData{} + host := models.Host{} + node := models.Node{} + nodes := []models.Node{} + server := models.ServerConfig{} err := json.NewDecoder(r.Body).Decode(&data) if err != nil { logger.Log(0, r.Header.Get("user"), "error decoding request body: ", err.Error()) logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest")) return } - - var networksToAdd = []string{} - for i := range data.LegacyNodes { - legacyNode := data.LegacyNodes[i] - record, err := database.FetchRecord(database.NODES_TABLE_NAME, legacyNode.ID) + for i, legacy := range data.LegacyNodes { + record, err := database.FetchRecord(database.NODES_TABLE_NAME, legacy.ID) if err != nil { - logger.Log(0, "no record for legacy node", legacyNode.ID, err.Error()) - continue - } else { - var oldLegacyNode models.LegacyNode - if err = json.Unmarshal([]byte(record), &oldLegacyNode); err != nil { - logger.Log(0, "error decoding legacy node", err.Error()) + slog.Error("legacy node not found", "error", err) + logic.ReturnErrorResponse(w, r, logic.FormatError(fmt.Errorf("legacy node not found %w", err), "badrequest")) + return + } + var legacyNode models.LegacyNode + if err = json.Unmarshal([]byte(record), &legacyNode); err != nil { + slog.Error("decoding legacy node", "errror", err) + logic.ReturnErrorResponse(w, r, logic.FormatError(fmt.Errorf("decode legacy node %w", err), "badrequest")) + return + } + if err := bcrypt.CompareHashAndPassword([]byte(legacyNode.Password), []byte(legacy.Password)); err != nil { + slog.Error("legacy node invalid password", "error", err) + logic.ReturnErrorResponse(w, r, logic.FormatError(fmt.Errorf("invalid password %w", err), "unauthorized")) + return + } + if i == 0 { + host, node = convertLegacyHostNode(legacy) + host.Name = data.HostName + host.HostPass = data.Password + host.OS = data.OS + if err := logic.CreateHost(&host); err != nil { + slog.Error("create host", "error", err) logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest")) - continue + return + } + server = servercfg.GetServerInfo() + if servercfg.GetBrokerType() == servercfg.EmqxBrokerType { + server.MQUserName = host.ID.String() } - if err := bcrypt.CompareHashAndPassword([]byte(oldLegacyNode.Password), []byte(legacyNode.Password)); err != nil { - logger.Log(0, "error decoding legacy password", err.Error()) - logic.ReturnErrorResponse(w, r, logic.FormatError(err, "unauthorized")) - continue + key, keyErr := logic.RetrievePublicTrafficKey() + if keyErr != nil { + slog.Error("retrieving traffickey", "error", err) + logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal")) + return } - networksToAdd = append(networksToAdd, oldLegacyNode.Network) - _ = database.DeleteRecord(database.NODES_TABLE_NAME, oldLegacyNode.ID) + server.TrafficKey = key + } else { + node = convertLegacyNode(legacyNode, host.ID) } - } - if len(networksToAdd) == 0 { - logger.Log(0, "no valid networks to migrate for host", data.NewHost.Name) - logic.ReturnErrorResponse(w, r, logic.FormatError(err, "unauthorized")) - return - } - if !logic.HostExists(&data.NewHost) { - logic.CheckHostPorts(&data.NewHost) - if err = logic.CreateHost(&data.NewHost); err != nil { - logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest")) - return + if err := logic.UpsertNode(&node); err != nil { + slog.Error("update node", "error", err) + continue } + host.Nodes = append(host.Nodes, node.ID.String()) + + nodes = append(nodes, node) } - key, keyErr := logic.RetrievePublicTrafficKey() - if keyErr != nil { - logger.Log(0, "error retrieving key:", keyErr.Error()) - logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal")) - return + if err := logic.UpsertHost(&host); err != nil { + slog.Error("save host", "error", err) } - server := servercfg.GetServerInfo() - server.TrafficKey = key - response := models.RegisterResponse{ - ServerConf: server, - RequestedHost: data.NewHost, + go mq.PublishPeerUpdate() + response := models.HostPull{ + Host: host, + Nodes: nodes, + ServerConfig: server, } w.WriteHeader(http.StatusOK) json.NewEncoder(w).Encode(&response) - logger.Log(0, "successfully migrated host", data.NewHost.Name, data.NewHost.ID.String()) - // notify host of changes, peer and node updates - go auth.CheckNetRegAndHostUpdate(networksToAdd, &data.NewHost) + + slog.Info("migrated nodes") + // check for gateways + for _, node := range data.LegacyNodes { + if node.IsEgressGateway == "yes" { + egressGateway := models.EgressGatewayRequest{ + NodeID: node.ID, + Ranges: node.EgressGatewayRanges, + NatEnabled: node.EgressGatewayNatEnabled, + } + if _, err := logic.CreateEgressGateway(egressGateway); err != nil { + logger.Log(0, "error creating egress gateway for node", node.ID, err.Error()) + } + } + if node.IsIngressGateway == "yes" { + ingressGateway := models.IngressRequest{} + ingressNode, err := logic.CreateIngressGateway(node.Network, node.ID, ingressGateway) + if err != nil { + logger.Log(0, "error creating ingress gateway for node", node.ID, err.Error()) + } + runUpdates(&ingressNode, true) + } + } +} + +func convertLegacyHostNode(legacy models.LegacyNode) (models.Host, models.Node) { + //convert host + host := models.Host{} + host.ID = uuid.New() + host.IPForwarding = models.ParseBool(legacy.IPForwarding) + host.AutoUpdate = servercfg.AutoUpdateEnabled() + host.Interface = "netmaker" + host.ListenPort = int(legacy.ListenPort) + host.MTU = int(legacy.MTU) + host.PublicKey, _ = wgtypes.ParseKey(legacy.PublicKey) + host.MacAddress = net.HardwareAddr(legacy.MacAddress) + host.TrafficKeyPublic = legacy.TrafficKeys.Mine + host.Nodes = append([]string{}, legacy.ID) + host.Interfaces = legacy.Interfaces + //host.DefaultInterface = legacy.Defaul + host.EndpointIP = net.ParseIP(legacy.Endpoint) + host.IsDocker = models.ParseBool(legacy.IsDocker) + host.IsK8S = models.ParseBool(legacy.IsK8S) + host.IsStatic = models.ParseBool(legacy.IsStatic) + node := convertLegacyNode(legacy, host.ID) + return host, node +} + +func convertLegacyNode(legacy models.LegacyNode, hostID uuid.UUID) models.Node { + //convert node + node := models.Node{} + node.ID, _ = uuid.Parse(legacy.ID) + node.HostID = hostID + node.Network = legacy.Network + valid4 := true + valid6 := true + _, cidr4, err := net.ParseCIDR(legacy.NetworkSettings.AddressRange) + if err != nil { + valid4 = false + slog.Warn("parsing address range", "error", err) + } else { + node.NetworkRange = *cidr4 + } + _, cidr6, err := net.ParseCIDR(legacy.NetworkSettings.AddressRange6) + if err != nil { + valid6 = false + slog.Warn("parsing address range6", "error", err) + } else { + node.NetworkRange6 = *cidr6 + } + node.Server = servercfg.GetServer() + node.Connected = models.ParseBool(legacy.Connected) + if valid4 { + node.Address = net.IPNet{ + IP: net.ParseIP(legacy.Address), + Mask: cidr4.Mask, + } + } + if valid6 { + node.Address6 = net.IPNet{ + IP: net.ParseIP(legacy.Address6), + Mask: cidr6.Mask, + } + } + node.Action = models.NODE_NOOP + node.LocalAddress = net.IPNet{ + IP: net.ParseIP(legacy.LocalAddress), + } + node.IsEgressGateway = models.ParseBool(legacy.IsEgressGateway) + node.EgressGatewayRanges = legacy.EgressGatewayRanges + node.IsIngressGateway = models.ParseBool(legacy.IsIngressGateway) + node.IsRelayed = false + node.IsRelay = false + node.RelayedNodes = []string{} + node.DNSOn = models.ParseBool(legacy.DNSOn) + node.PersistentKeepalive = time.Duration(legacy.PersistentKeepalive) + node.LastModified = time.Now() + node.ExpirationDateTime, _ = time.Parse(strconv.Itoa(int(legacy.ExpirationDateTime)), "0") + node.EgressGatewayNatEnabled = models.ParseBool(legacy.EgressGatewayNatEnabled) + node.EgressGatewayRequest = legacy.EgressGatewayRequest + node.IngressGatewayRange = legacy.IngressGatewayRange + node.IngressGatewayRange6 = legacy.IngressGatewayRange6 + node.DefaultACL = legacy.DefaultACL + node.OwnerID = legacy.OwnerID + node.FailoverNode, _ = uuid.Parse(legacy.FailoverNode) + node.Failover = models.ParseBool(legacy.Failover) + return node } diff --git a/models/migrate.go b/models/migrate.go index 9972c22c1..978354cb4 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -2,6 +2,8 @@ package models // MigrationData struct needed to create new v0.18.0 node from v.0.17.X node type MigrationData struct { - NewHost Host + HostName string + Password string + OS string LegacyNodes []LegacyNode } diff --git a/models/node.go b/models/node.go index ee10f5597..ed778d00e 100644 --- a/models/node.go +++ b/models/node.go @@ -100,7 +100,6 @@ type Node struct { // LegacyNode - legacy struct for node model type LegacyNode struct { ID string `json:"id,omitempty" bson:"id,omitempty" yaml:"id,omitempty" validate:"required,min=5,id_unique"` - HostID string `json:"hostid,omitempty" bson:"id,omitempty" yaml:"hostid,omitempty" validate:"required,min=5,id_unique"` Address string `json:"address" bson:"address" yaml:"address" validate:"omitempty,ipv4"` Address6 string `json:"address6" bson:"address6" yaml:"address6" validate:"omitempty,ipv6"` LocalAddress string `json:"localaddress" bson:"localaddress" yaml:"localaddress" validate:"omitempty"` @@ -109,7 +108,6 @@ type LegacyNode struct { NetworkSettings Network `json:"networksettings" bson:"networksettings" yaml:"networksettings" validate:"-"` ListenPort int32 `json:"listenport" bson:"listenport" yaml:"listenport" validate:"omitempty,numeric,min=1024,max=65535"` LocalListenPort int32 `json:"locallistenport" bson:"locallistenport" yaml:"locallistenport" validate:"numeric,min=0,max=65535"` - ProxyListenPort int32 `json:"proxy_listen_port" bson:"proxy_listen_port" yaml:"proxy_listen_port" validate:"numeric,min=0,max=65535"` PublicKey string `json:"publickey" bson:"publickey" yaml:"publickey" validate:"required,base64"` Endpoint string `json:"endpoint" bson:"endpoint" yaml:"endpoint" validate:"required,ip"` AllowedIPs []string `json:"allowedips" bson:"allowedips" yaml:"allowedips"` @@ -153,8 +151,6 @@ type LegacyNode struct { FirewallInUse string `json:"firewallinuse" bson:"firewallinuse" yaml:"firewallinuse"` InternetGateway string `json:"internetgateway" bson:"internetgateway" yaml:"internetgateway"` Connected string `json:"connected" bson:"connected" yaml:"connected" validate:"checkyesorno"` - PendingDelete bool `json:"pendingdelete" bson:"pendingdelete" yaml:"pendingdelete"` - Proxy bool `json:"proxy" bson:"proxy" yaml:"proxy"` // == PRO == DefaultACL string `json:"defaultacl,omitempty" bson:"defaultacl,omitempty" yaml:"defaultacl,omitempty" validate:"checkyesornoorunset"` OwnerID string `json:"ownerid,omitempty" bson:"ownerid,omitempty" yaml:"ownerid,omitempty"` @@ -527,7 +523,7 @@ func (ln *LegacyNode) ConvertToNewNode() (*Host, *Node) { func (n *Node) Legacy(h *Host, s *ServerConfig, net *Network) *LegacyNode { l := LegacyNode{} l.ID = n.ID.String() - l.HostID = h.ID.String() + //l.HostID = h.ID.String() l.Address = n.Address.String() l.Address6 = n.Address6.String() l.Interfaces = h.Interfaces diff --git a/scripts/nm-upgrade.sh b/scripts/nm-upgrade.sh new file mode 100755 index 000000000..7da1ce6e9 --- /dev/null +++ b/scripts/nm-upgrade.sh @@ -0,0 +1,574 @@ +#!/bin/bash + +CONFIG_FILE=netmaker.env +# location of nm-quick.sh (usually `/root`) +SCRIPT_DIR=$(dirname "$(realpath "$0")") +CONFIG_PATH="$SCRIPT_DIR/$CONFIG_FILE" +NM_QUICK_VERSION="0.1.0" +LATEST=$(curl -s https://api.github.com/repos/gravitl/netmaker/releases/latest | grep "tag_name" | cut -d : -f 2,3 | tr -d [:space:],\") + +if [ "$(id -u)" -ne 0 ]; then + echo "This script must be run as root" + exit 1 +fi + +unset INSTALL_TYPE +unset NETMAKER_BASE_DOMAIN + +# usage - displays usage instructions +usage() { + echo "nm-upgrade.sh v$NM_QUICK_VERSION" + echo "usage: ./nm-upgrade.sh" + exit 1 +} + +while getopts v flag; do + case "${flag}" in + v) + usage + exit 0 + ;; + *) + usage + exit 0 + ;; + esac +done + +# print_logo - prints the netmaker logo +print_logo() { + cat <<"EOF" +- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + __ __ ______ ______ __ __ ______ __ __ ______ ______ +/\ "-.\ \ /\ ___\ /\__ _\ /\ "-./ \ /\ __ \ /\ \/ / /\ ___\ /\ == \ +\ \ \-. \ \ \ __\ \/_/\ \/ \ \ \-./\ \ \ \ __ \ \ \ _"-. \ \ __\ \ \ __< + \ \_\\"\_\ \ \_____\ \ \_\ \ \_\ \ \_\ \ \_\ \_\ \ \_\ \_\ \ \_____\ \ \_\ \_\ + \/_/ \/_/ \/_____/ \/_/ \/_/ \/_/ \/_/\/_/ \/_/\/_/ \/_____/ \/_/ /_/ + + +- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +EOF +} + +# set_buildinfo - sets the information based on script input for how the installation should be run +set_buildinfo() { + + MASTERKEY=$(grep MASTER_KEY docker-compose.yml | awk '{print $2;}' | tr -d '"') + EMAIL=$(grep email Caddyfile | awk '{print $2;}' | tr -d '"') + BROKER=$(grep SERVER_NAME docker-compose.yml | awk '{print $2;}' | tr -d '"') + PREFIX="broker." + NETMAKER_BASE_DOMAIN=${BROKER/#$PREFIX} + + + echo "-----------------------------------------------------" + echo "Would you like to install Netmaker Community Edition (CE), or Netmaker Enterprise Edition (EE)?" + echo "EE will require you to create an account at https://app.netmaker.io" + echo "-----------------------------------------------------" + select install_option in "Community Edition" "Enterprise Edition"; do + case $REPLY in + 1) + echo "installing Netmaker CE" + INSTALL_TYPE="ce" + break + ;; + 2) + echo "installing Netmaker EE" + INSTALL_TYPE="ee" + break + ;; + *) echo "invalid option $REPLY" ;; + esac + done + + echo "-----------Build Options-----------------------------" + echo " EE or CE: $INSTALL_TYPE" + echo " Version: $LATEST" + echo " Installer: v$NM_QUICK_VERSION" + echo "-----------------------------------------------------" + +} + +# install_yq - install yq if not present +install_yq() { + if ! command -v yq &>/dev/null; then + wget -qO /usr/bin/yq https://github.com/mikefarah/yq/releases/download/v4.31.1/yq_linux_$(dpkg --print-architecture) + chmod +x /usr/bin/yq + fi + set +e + if ! command -v yq &>/dev/null; then + set -e + wget -qO /usr/bin/yq https://github.com/mikefarah/yq/releases/download/v4.31.1/yq_linux_amd64 + chmod +x /usr/bin/yq + fi + set -e + if ! command -v yq &>/dev/null; then + echo "failed to install yq. Please install yq and try again." + echo "https://github.com/mikefarah/yq/#install" + exit 1 + fi +} + +# install and run upgrade tool +upgrade() { + wget -qO /tmp/nm-upgrade https://fileserver.netmaker.io/upgrade/nm-upgrade-${ARCH} + chmod +x /tmp/nm-upgrade + echo "generating netclient configuration files" + /tmp/nm-upgrade +} + +# setup_netclient - installs netclient +setup_netclient() { + wget -qO netclient https://github.com/gravitl/netclient/releases/download/$LATEST/netclient-linux-$ARCH + chmod +x netclient + ./netclient install -v 3 +} + + +# wait_seconds - wait for the specified period of time +wait_seconds() { ( + for ((a = 1; a <= $1; a++)); do + echo ". . ." + sleep 1 + done +); } + +# confirm - get user input to confirm that they want to perform the next step +confirm() { ( + while true; do + read -p 'Does everything look right? [y/n]: ' yn + case $yn in + [Yy]*) + override="true" + break + ;; + [Nn]*) + echo "exiting..." + exit 1 + # TODO start from the beginning instead + ;; + *) echo "Please answer yes or no." ;; + esac + done +) } + +save_config() { ( + echo "Saving the config to $CONFIG_PATH" + touch "$CONFIG_PATH" + save_config_item NM_EMAIL "$EMAIL" + save_config_item NM_DOMAIN "$NETMAKER_BASE_DOMAIN" + save_config_item UI_IMAGE_TAG "$LATEST" + # version-specific entries + if [ "$INSTALL_TYPE" = "ee" ]; then + save_config_item NETMAKER_TENANT_ID "$TENANT_ID" + save_config_item LICENSE_KEY "$LICENSE_KEY" + save_config_item METRICS_EXPORTER "on" + save_config_item PROMETHEUS "on" + ave_config_item SERVER_IMAGE_TAG "$LATEST-ee" + else + save_config_item METRICS_EXPORTER "off" + save_config_item PROMETHEUS "off" + save_config_item SERVER_IMAGE_TAG "$LATEST" + fi + # copy entries from the previous config + local toCopy=("SERVER_HOST" "MASTER_KEY" "TURN_USERNAME" "TURN_PASSWORD" "MQ_USERNAME" "MQ_PASSWORD" + "INSTALL_TYPE" "NODE_ID" "DNS_MODE" "NETCLIENT_AUTO_UPDATE" "API_PORT" + "CORS_ALLOWED_ORIGIN" "DISPLAY_KEYS" "DATABASE" "SERVER_BROKER_ENDPOINT" "STUN_PORT" "VERBOSITY" + "TURN_PORT" "USE_TURN" "DEBUG_MODE" "TURN_API_PORT" "REST_BACKEND" + "DISABLE_REMOTE_IP_CHECK" "NETCLIENT_ENDPOINT_DETECTION" "TELEMETRY" "AUTH_PROVIDER" "CLIENT_ID" "CLIENT_SECRET" + "FRONTEND_URL" "AZURE_TENANT" "OIDC_ISSUER" "EXPORTER_API_PORT") + for name in "${toCopy[@]}"; do + save_config_item $name "${!name}" + done + # preserve debug entries + if test -n "$NM_SKIP_BUILD"; then + save_config_item NM_SKIP_BUILD "$NM_SKIP_BUILD" + fi + if test -n "$NM_SKIP_CLONE"; then + save_config_item NM_SKIP_CLONE "$NM_SKIP_CLONE" + fi + if test -n "$NM_SKIP_DEPS"; then + save_config_item NM_SKIP_DEPS "$NM_SKIP_DEPS" + fi +); } + +save_config_item() { ( + local NAME="$1" + local VALUE="$2" + #echo "$NAME=$VALUE" + if test -z "$VALUE"; then + # load the default for empty values + VALUE=$(awk -F'=' "/^$NAME/ { print \$2}" "$SCRIPT_DIR/netmaker.default.env") + # trim quotes for docker + VALUE=$(echo "$VALUE" | sed -E "s|^(['\"])(.*)\1$|\2|g") + #echo "Default for $NAME=$VALUE" + fi + # TODO single quote passwords + if grep -q "^$NAME=" "$CONFIG_PATH"; then + # TODO escape | in the value + sed -i "s|$NAME=.*|$NAME=$VALUE|" "$CONFIG_PATH" + else + echo "$NAME=$VALUE" >>"$CONFIG_PATH" + fi +); } + +# install_dependencies - install necessary packages to run netmaker +install_dependencies() { + + if test -n "$NM_SKIP_DEPS"; then + return + fi + + echo "checking dependencies..." + + OS=$(uname) + if [ -f /etc/debian_version ]; then + dependencies="git wireguard wireguard-tools dnsutils jq docker.io docker-compose grep gawk" + update_cmd='apt update' + install_cmd='apt-get install -y' + elif [ -f /etc/alpine-release ]; then + dependencies="git wireguard jq docker.io docker-compose grep gawk" + update_cmd='apk update' + install_cmd='apk --update add' + elif [ -f /etc/centos-release ]; then + dependencies="git wireguard jq bind-utils docker.io docker-compose grep gawk" + update_cmd='yum update' + install_cmd='yum install -y' + elif [ -f /etc/fedora-release ]; then + dependencies="git wireguard bind-utils jq docker.io docker-compose grep gawk" + update_cmd='dnf update' + install_cmd='dnf install -y' + elif [ -f /etc/redhat-release ]; then + dependencies="git wireguard jq docker.io bind-utils docker-compose grep gawk" + update_cmd='yum update' + install_cmd='yum install -y' + elif [ -f /etc/arch-release ]; then + dependencies="git wireguard-tools dnsutils jq docker.io docker-compose grep gawk" + update_cmd='pacman -Sy' + install_cmd='pacman -S --noconfirm' + elif [ "${OS}" = "FreeBSD" ]; then + dependencies="git wireguard wget jq docker.io docker-compose grep gawk" + update_cmd='pkg update' + install_cmd='pkg install -y' + else + install_cmd='' + fi + + if [ -z "${install_cmd}" ]; then + echo "OS unsupported for automatic dependency install" + # TODO shouldnt exit, check if deps available, if not + # ask the user to install manually and continue when ready + exit 1 + fi + # TODO add other supported architectures + ARCH=$(uname -m) + if [ "$ARCH" = "x86_64" ]; then + ARCH=amd64 + elif [ "$ARCH" = "aarch64" ]; then + ARCH=arm64 + else + echo "Unsupported architechure" + # exit 1 + fi + set -- $dependencies + + ${update_cmd} + + while [ -n "$1" ]; do + if [ "${OS}" = "FreeBSD" ]; then + is_installed=$(pkg check -d $1 | grep "Checking" | grep "done") + if [ "$is_installed" != "" ]; then + echo " " $1 is installed + else + echo " " $1 is not installed. Attempting install. + ${install_cmd} $1 + sleep 5 + is_installed=$(pkg check -d $1 | grep "Checking" | grep "done") + if [ "$is_installed" != "" ]; then + echo " " $1 is installed + elif [ -x "$(command -v $1)" ]; then + echo " " $1 is installed + else + echo " " FAILED TO INSTALL $1 + echo " " This may break functionality. + fi + fi + else + if [ "${OS}" = "OpenWRT" ] || [ "${OS}" = "TurrisOS" ]; then + is_installed=$(opkg list-installed $1 | grep $1) + else + is_installed=$(dpkg-query -W --showformat='${Status}\n' $1 | grep "install ok installed") + fi + if [ "${is_installed}" != "" ]; then + echo " " $1 is installed + else + echo " " $1 is not installed. Attempting install. + ${install_cmd} $1 + sleep 5 + if [ "${OS}" = "OpenWRT" ] || [ "${OS}" = "TurrisOS" ]; then + is_installed=$(opkg list-installed $1 | grep $1) + else + is_installed=$(dpkg-query -W --showformat='${Status}\n' $1 | grep "install ok installed") + fi + if [ "${is_installed}" != "" ]; then + echo " " $1 is installed + elif [ -x "$(command -v $1)" ]; then + echo " " $1 is installed + else + echo " " FAILED TO INSTALL $1 + echo " " This may break functionality. + fi + fi + fi + shift + done + + echo "-----------------------------------------------------" + echo "dependency check complete" + echo "-----------------------------------------------------" +} + +# set_install_vars - sets the variables that will be used throughout installation +set_install_vars() { + + IP_ADDR=$(dig -4 myip.opendns.com @resolver1.opendns.com +short) + if [ "$IP_ADDR" = "" ]; then + IP_ADDR=$(curl -s ifconfig.me) + fi + if [ "$NETMAKER_BASE_DOMAIN" = "" ]; then + NETMAKER_BASE_DOMAIN=nm.$(echo $IP_ADDR | tr . -).nip.io + fi + SERVER_HOST=$IP_ADDR + if test -z "$MASTER_KEY"; then + MASTER_KEY=$( + tr -dc A-Za-z0-9 &1) + + if [[ "$i" == 8 ]]; then + echo " Caddy is having an issue setting up certificates, please investigate (docker logs caddy)" + echo " Exiting..." + exit 1 + elif [[ "$curlresponse" == *"failed to verify the legitimacy of the server"* ]]; then + echo " Certificates not yet configured, retrying..." + + elif [[ "$curlresponse" == *"left intact"* ]]; then + echo " Certificates ok" + break + else + secs=$(($i * 5 + 10)) + echo " Issue establishing connection...retrying in $secs seconds..." + fi + sleep $secs + done + +} + +# print_success - prints a success message upon completion +print_success() { + echo "-----------------------------------------------------------------" + echo "-----------------------------------------------------------------" + echo "Netmaker setup is now complete. You are ready to begin using Netmaker." + echo "Visit dashboard.$NETMAKER_BASE_DOMAIN to log in" + echo "-----------------------------------------------------------------" + echo "-----------------------------------------------------------------" +} + +cleanup() { + echo "Stopping all containers..." + local containers=("mq" "netmaker-ui" "coredns" "turn" "caddy" "netmaker" "netmaker-exporter" "prometheus" "grafana") + for name in "${containers[@]}"; do + local running=$(docker ps | grep -w "$name") + local exists=$(docker ps -a | grep -w "$name") + if test -n "$running"; then + docker stop "$name" 1>/dev/null + fi + if test -n "$exists"; then + docker rm "$name" 1>/dev/null + fi + done +} + +# print netmaker logo +print_logo + +# read the config +if [ -f "$CONFIG_PATH" ]; then + echo "Using config: $CONFIG_PATH" + source "$CONFIG_PATH" + if [ "$UPGRADE_FLAG" = "yes" ]; then + INSTALL_TYPE="ee" + fi +fi + +# setup the build instructions +set_buildinfo + +set +e + +# install necessary packages +install_dependencies + +# install yq if necessary +install_yq + +set -e + +# get user input for variables +set_install_vars + +set +e +cleanup +set -e + +# get upgrade tool and run +upgrade + +# get and set config files, startup docker-compose +install_netmaker + +set +e + +# make sure Caddy certs are working +test_connection + +set -e + +# install netclient +setup_netclient + + +# print success message +print_success + From 8ce4cec124146c5aec23d097db79f35ff7446f03 Mon Sep 17 00:00:00 2001 From: Matthew R Kasun Date: Wed, 16 Aug 2023 01:30:49 -0400 Subject: [PATCH 02/11] Net 406 (#2514) * create gateways during migration * set version for testing * restruct migration * debug logging * enforce unique names for ext client names (#2476) * enforce unique names for ext client names * only check for unique id on creation * check for unique id if changed * prune(NET-483): remove defunct host.internetgateway field (#2487) * don't reference host on err (#2493) * deprecrate netclient install scripts (#2490) * Net 500: validate network parameter passed to node endpoints (#2480) * enforce unique names for ext client names * only check for unique id on creation * check for unique id if changed * validate network parameter passed to node endpoints --------- Co-authored-by: Abhishek K <32607604+abhishek9686@users.noreply.github.com> * NET-513 (#2492) nm-certs.sh now requests certificate for EE and CE edition domains accordingly. * [NET-404] Run in limited mode when ee checks fail (#2474) * Add limited http handlers functionality to rest handler * Export ee.errValidation (ee.ErrValidation) * Export a fatal error handled by the hook manager * Export a new status variable for unlicensed server * Mark server as unlicensed when ee checks fail * Handle license validation failures with a (re)boot in a limited state * Revert "Export a fatal error handled by the hook manager" This reverts commit 069c21974a8d36e889c73ad78023448d787d62a5. * Revert "Export ee.errValidation (ee.ErrValidation)" This reverts commit 59dbab8c79773ca5d879f28cbaf53f3dd4297b9b. * Revert "Add limited http handlers functionality to rest handler" This reverts commit e2f1f28facaca54713db76a588839cd2733cf673. * Revert "Handle license validation failures with a (re)boot in a limited state" This reverts commit 58cfbbaf522a1345aac1fa67964ebff0a6d60cd8. * Revert "Mark server as unlicensed when ee checks fail" This reverts commit 77c6dbdd3c9cfa6e7d6becedef6251e8617ae367. * Handle license validation failures with a middleware * Forbid responses if unlicensed ee and not in status api * Remove unused func * feat(NET-449): add sync feature to request a host pull from server (#2491) * fix(NET-486): change client name length validation (#2498) set limit to 5<=x<=32 * [NET-477] Pick AMB URL dynamically (#2489) * Introduce config for environment * Introduce func to get environment * Choose accounts api host from environment * Test the ee package on workflows * Use build tag ee for license_test.go * [Feature]: nm-quick script tackling arm TODO support (#2488) * domain flag for auto installs * use static servers with custom domain (#2421) * send delete peer update always * fix add/remove host api calls * keep mq updates in a single go func * move branch test logic to devops (#2443) * handle IOT OS * save server name to env (#2460) * ensure branch test servers available after test runs (#2467) * save server name to env * free server always; add PR to discord messages * use correct method to delete droplets (#2468) * quick fix for the launcher * removed exit when triggering not supported exit and removed the TODO comments related to this issue --------- Co-authored-by: Matthew R Kasun Co-authored-by: Alex Feiszli <31018251+afeiszli@users.noreply.github.com> Co-authored-by: Christopher Blaha Co-authored-by: Abhishek Kondur Co-authored-by: Abhishek K <32607604+abhishek9686@users.noreply.github.com> * rebase conflict * include pass and os in mirgration data * node network ranges * remove debugging logs * add gateways * use sent node * upgrade shell script * associate node to host during migration * add node to host.Nodes and publish peer update * save host outside loop * fix script name * simplify upgrade script * don't migrate relays * simplify upgrade script even more * guard against blank address or address6 * typos * fix convertsion of persistent keepalive * remove weird paste * another paste error --------- Co-authored-by: Aceix Co-authored-by: Abhishek K <32607604+abhishek9686@users.noreply.github.com> Co-authored-by: Farukh Khan Co-authored-by: Gabriel de Souza Seibel Co-authored-by: bornav <51048565+bornav@users.noreply.github.com> Co-authored-by: Alex Feiszli <31018251+afeiszli@users.noreply.github.com> Co-authored-by: Christopher Blaha Co-authored-by: Abhishek Kondur --- controllers/migrate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/migrate.go b/controllers/migrate.go index fa03fccf9..6bf2e540f 100644 --- a/controllers/migrate.go +++ b/controllers/migrate.go @@ -199,7 +199,7 @@ func convertLegacyNode(legacy models.LegacyNode, hostID uuid.UUID) models.Node { node.IsRelay = false node.RelayedNodes = []string{} node.DNSOn = models.ParseBool(legacy.DNSOn) - node.PersistentKeepalive = time.Duration(legacy.PersistentKeepalive) + node.PersistentKeepalive = time.Duration(int64(time.Second) * int64(legacy.PersistentKeepalive)) node.LastModified = time.Now() node.ExpirationDateTime, _ = time.Parse(strconv.Itoa(int(legacy.ExpirationDateTime)), "0") node.EgressGatewayNatEnabled = models.ParseBool(legacy.EgressGatewayNatEnabled) From daaf488216d48ffa18f4e76cc28c5f6216caca0f Mon Sep 17 00:00:00 2001 From: Abhishek K <32607604+abhishek9686@users.noreply.github.com> Date: Wed, 16 Aug 2023 16:43:29 +0530 Subject: [PATCH 03/11] NET-406: fix typo in upgrade script (#2515) * create gateways during migration * set version for testing * restruct migration * debug logging * enforce unique names for ext client names (#2476) * enforce unique names for ext client names * only check for unique id on creation * check for unique id if changed * prune(NET-483): remove defunct host.internetgateway field (#2487) * don't reference host on err (#2493) * deprecrate netclient install scripts (#2490) * Net 500: validate network parameter passed to node endpoints (#2480) * enforce unique names for ext client names * only check for unique id on creation * check for unique id if changed * validate network parameter passed to node endpoints --------- Co-authored-by: Abhishek K <32607604+abhishek9686@users.noreply.github.com> * NET-513 (#2492) nm-certs.sh now requests certificate for EE and CE edition domains accordingly. * [NET-404] Run in limited mode when ee checks fail (#2474) * Add limited http handlers functionality to rest handler * Export ee.errValidation (ee.ErrValidation) * Export a fatal error handled by the hook manager * Export a new status variable for unlicensed server * Mark server as unlicensed when ee checks fail * Handle license validation failures with a (re)boot in a limited state * Revert "Export a fatal error handled by the hook manager" This reverts commit 069c21974a8d36e889c73ad78023448d787d62a5. * Revert "Export ee.errValidation (ee.ErrValidation)" This reverts commit 59dbab8c79773ca5d879f28cbaf53f3dd4297b9b. * Revert "Add limited http handlers functionality to rest handler" This reverts commit e2f1f28facaca54713db76a588839cd2733cf673. * Revert "Handle license validation failures with a (re)boot in a limited state" This reverts commit 58cfbbaf522a1345aac1fa67964ebff0a6d60cd8. * Revert "Mark server as unlicensed when ee checks fail" This reverts commit 77c6dbdd3c9cfa6e7d6becedef6251e8617ae367. * Handle license validation failures with a middleware * Forbid responses if unlicensed ee and not in status api * Remove unused func * feat(NET-449): add sync feature to request a host pull from server (#2491) * fix(NET-486): change client name length validation (#2498) set limit to 5<=x<=32 * [NET-477] Pick AMB URL dynamically (#2489) * Introduce config for environment * Introduce func to get environment * Choose accounts api host from environment * Test the ee package on workflows * Use build tag ee for license_test.go * [Feature]: nm-quick script tackling arm TODO support (#2488) * domain flag for auto installs * use static servers with custom domain (#2421) * send delete peer update always * fix add/remove host api calls * keep mq updates in a single go func * move branch test logic to devops (#2443) * handle IOT OS * save server name to env (#2460) * ensure branch test servers available after test runs (#2467) * save server name to env * free server always; add PR to discord messages * use correct method to delete droplets (#2468) * quick fix for the launcher * removed exit when triggering not supported exit and removed the TODO comments related to this issue --------- Co-authored-by: Matthew R Kasun Co-authored-by: Alex Feiszli <31018251+afeiszli@users.noreply.github.com> Co-authored-by: Christopher Blaha Co-authored-by: Abhishek Kondur Co-authored-by: Abhishek K <32607604+abhishek9686@users.noreply.github.com> * rebase conflict * include pass and os in mirgration data * node network ranges * remove debugging logs * add gateways * use sent node * upgrade shell script * associate node to host during migration * add node to host.Nodes and publish peer update * save host outside loop * fix script name * simplify upgrade script * don't migrate relays * simplify upgrade script even more * guard against blank address or address6 * typos * fix convertsion of persistent keepalive * remove weird paste * another paste error * fix command typo --------- Co-authored-by: Matthew R Kasun Co-authored-by: Aceix Co-authored-by: Farukh Khan Co-authored-by: Gabriel de Souza Seibel Co-authored-by: bornav <51048565+bornav@users.noreply.github.com> Co-authored-by: Alex Feiszli <31018251+afeiszli@users.noreply.github.com> Co-authored-by: Christopher Blaha --- scripts/nm-upgrade.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/nm-upgrade.sh b/scripts/nm-upgrade.sh index 7da1ce6e9..34a2df998 100755 --- a/scripts/nm-upgrade.sh +++ b/scripts/nm-upgrade.sh @@ -168,7 +168,7 @@ save_config() { ( save_config_item LICENSE_KEY "$LICENSE_KEY" save_config_item METRICS_EXPORTER "on" save_config_item PROMETHEUS "on" - ave_config_item SERVER_IMAGE_TAG "$LATEST-ee" + save_config_item SERVER_IMAGE_TAG "$LATEST-ee" else save_config_item METRICS_EXPORTER "off" save_config_item PROMETHEUS "off" From f6dded0f2d0852b41fc126690bb69348b2d1cfbf Mon Sep 17 00:00:00 2001 From: Abhishek K <32607604+abhishek9686@users.noreply.github.com> Date: Wed, 16 Aug 2023 19:37:46 +0530 Subject: [PATCH 04/11] revert license response to limits tag for backward compatibility (#2519) * revert license response to limits tag for backward compatibility * revert tags --- ee/types.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ee/types.go b/ee/types.go index 355bc6d3a..1bae0bb10 100644 --- a/ee/types.go +++ b/ee/types.go @@ -26,13 +26,13 @@ var errValidation = fmt.Errorf(license_validation_err_msg) type LicenseKey struct { LicenseValue string `json:"license_value"` // actual (public) key and the unique value for the key Expiration int64 `json:"expiration"` - UsageServers int `json:"usage_servers"` - UsageUsers int `json:"usage_users"` - UsageClients int `json:"usage_clients"` - UsageHosts int `json:"usage_hosts"` - UsageNetworks int `json:"usage_networks"` - UsageIngresses int `json:"usage_ingresses"` - UsageEgresses int `json:"usage_egresses"` + UsageServers int `json:"limit_servers"` + UsageUsers int `json:"limit_users"` + UsageClients int `json:"limit_clients"` + UsageHosts int `json:"limit_hosts"` + UsageNetworks int `json:"limit_networks"` + UsageIngresses int `json:"limit_ingresses"` + UsageEgresses int `json:"limit_egresses"` Metadata string `json:"metadata"` IsActive bool `json:"is_active"` // yes if active } @@ -46,7 +46,7 @@ type ValidatedLicense struct { // LicenseSecret - the encrypted struct for sending user-id type LicenseSecret struct { AssociatedID string `json:"associated_id" binding:"required"` // UUID for user foreign key to User table - Usage Usage `json:"usage" binding:"required"` + Usage Usage `json:"limits" binding:"required"` } // Usage - struct for license usage From 2c1d437959d3229bcd9d2e6728e2e4d358852528 Mon Sep 17 00:00:00 2001 From: Abhishek K <32607604+abhishek9686@users.noreply.github.com> Date: Wed, 16 Aug 2023 20:09:51 +0530 Subject: [PATCH 05/11] V0.20.6 release notes (#2520) * update release.md * update release.md * update release.md * update release.md --- release.md | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/release.md b/release.md index 82f009f55..de1d29dc5 100644 --- a/release.md +++ b/release.md @@ -2,21 +2,18 @@ # Netmaker v0.20.6 ## Whats New -- Extclient Acls -- Force delete host along with all the associated nodes +- Sync clients with server state from UI ## What's Fixed -- Deprecated Proxy -- Solved Race condition for multiple nodes joining network at same time -- Node dns toggle -- Simplified Firewall rules for added stability +- Upgrade Process from v0.17.1 to latest version can be now done seamlessly, please refer docs for more information +- Expired nodes clean up is handled correctly now +- Ext client config generation fixed for ipv6 endpoints +- installation process will only generate certs required for required Domains based on CE or EE +- support for ARM machines on install script ## known issues -- Expired nodes are not getting cleaned up - Windows installer does not install WireGuard - netclient-gui will continously display error dialog if netmaker server is offline -- Incorrect metrics against ext clients -- Host ListenPorts set to 0 after migration from 0.17.1 -> 0.20.6 - Mac IPv6 addresses/route issues - Docker client can not re-join after complete deletion - netclient-gui network tab blank after disconnect From fe470e92dc9c873f33159e0dd4b6ab3b851ffc5e Mon Sep 17 00:00:00 2001 From: Christopher Blaha Date: Fri, 18 Aug 2023 02:25:12 -0400 Subject: [PATCH 06/11] update develop to 0.20.7 (#2522) * update develop to 0.20.7 * change version to 0.21.0 --- .github/ISSUE_TEMPLATE/bug-report.yml | 1 + README.md | 2 +- compose/docker-compose.netclient.yml | 2 +- controllers/docs.go | 2 +- k8s/client/netclient-daemonset.yaml | 2 +- k8s/client/netclient.yaml | 2 +- k8s/server/netmaker-ui.yaml | 2 +- main.go | 2 +- release.md | 2 +- scripts/nm-upgrade-0-17-1-to-0-19-0.sh | 2 +- swagger.yaml | 2 +- 11 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index 02c2b48c7..b57a4b134 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -31,6 +31,7 @@ body: label: Version description: What version are you running? options: + - v0.21.0 - v0.20.6 - v0.20.5 - v0.20.4 diff --git a/README.md b/README.md index 840ceb301..6ddb8b670 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@

- + diff --git a/compose/docker-compose.netclient.yml b/compose/docker-compose.netclient.yml index 9144b76cd..8c184fffb 100644 --- a/compose/docker-compose.netclient.yml +++ b/compose/docker-compose.netclient.yml @@ -3,7 +3,7 @@ version: "3.4" services: netclient: container_name: netclient - image: 'gravitl/netclient:v0.20.6' + image: 'gravitl/netclient:v0.21.0' hostname: netmaker-1 network_mode: host restart: on-failure diff --git a/controllers/docs.go b/controllers/docs.go index 592922982..2cbe329ea 100644 --- a/controllers/docs.go +++ b/controllers/docs.go @@ -10,7 +10,7 @@ // // Schemes: https // BasePath: / -// Version: 0.20.6 +// Version: 0.21.0 // Host: netmaker.io // // Consumes: diff --git a/k8s/client/netclient-daemonset.yaml b/k8s/client/netclient-daemonset.yaml index 635e25769..12c897564 100644 --- a/k8s/client/netclient-daemonset.yaml +++ b/k8s/client/netclient-daemonset.yaml @@ -16,7 +16,7 @@ spec: hostNetwork: true containers: - name: netclient - image: gravitl/netclient:v0.20.6 + image: gravitl/netclient:v0.21.0 env: - name: TOKEN value: "TOKEN_VALUE" diff --git a/k8s/client/netclient.yaml b/k8s/client/netclient.yaml index 12a6ea116..f329be78a 100644 --- a/k8s/client/netclient.yaml +++ b/k8s/client/netclient.yaml @@ -28,7 +28,7 @@ spec: # - "" containers: - name: netclient - image: gravitl/netclient:v0.20.6 + image: gravitl/netclient:v0.21.0 env: - name: TOKEN value: "TOKEN_VALUE" diff --git a/k8s/server/netmaker-ui.yaml b/k8s/server/netmaker-ui.yaml index 2d7e62ca5..042ea2fdb 100644 --- a/k8s/server/netmaker-ui.yaml +++ b/k8s/server/netmaker-ui.yaml @@ -15,7 +15,7 @@ spec: spec: containers: - name: netmaker-ui - image: gravitl/netmaker-ui:v0.20.6 + image: gravitl/netmaker-ui:v0.21.0 ports: - containerPort: 443 env: diff --git a/main.go b/main.go index a0fe2824f..4bf424873 100644 --- a/main.go +++ b/main.go @@ -29,7 +29,7 @@ import ( "golang.org/x/exp/slog" ) -var version = "v0.20.6" +var version = "v0.21.0" // Start DB Connection and start API Request Handler func main() { diff --git a/release.md b/release.md index de1d29dc5..1c363132c 100644 --- a/release.md +++ b/release.md @@ -1,5 +1,5 @@ -# Netmaker v0.20.6 +# Netmaker v0.21.0 ## Whats New - Sync clients with server state from UI diff --git a/scripts/nm-upgrade-0-17-1-to-0-19-0.sh b/scripts/nm-upgrade-0-17-1-to-0-19-0.sh index 359457441..7ece935ff 100644 --- a/scripts/nm-upgrade-0-17-1-to-0-19-0.sh +++ b/scripts/nm-upgrade-0-17-1-to-0-19-0.sh @@ -1,6 +1,6 @@ #!/bin/bash -LATEST="v0.20.6" +LATEST="v0.21.0" INSTALL_PATH="/root" trap restore_old_netmaker_instructions diff --git a/swagger.yaml b/swagger.yaml index c71d63877..77c7de77b 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -704,7 +704,7 @@ info: API calls must be authenticated via a header of the format -H “Authorization: Bearer ” There are two methods to obtain YOUR_SECRET_KEY: 1. Using the masterkey. By default, this value is “secret key,” but you should change this on your instance and keep it secure. This value can be set via env var at startup or in a config file (config/environments/< env >.yaml). See the [Netmaker](https://docs.netmaker.org/index.html) documentation for more details. 2. Using a JWT received for a node. This can be retrieved by calling the /api/nodes//authenticate endpoint, as documented below. title: Netmaker - version: 0.20.6 + version: 0.21.0 paths: /api/dns: get: From a775d7402f39cf56608bb5db02299dfdd66f30bd Mon Sep 17 00:00:00 2001 From: Abhishek K <32607604+abhishek9686@users.noreply.github.com> Date: Tue, 22 Aug 2023 11:48:10 +0530 Subject: [PATCH 07/11] set coredns to 1.10.1 version (#2527) --- compose/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index d17275291..7fbf71136 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -62,7 +62,7 @@ services: coredns: container_name: coredns - image: coredns/coredns + image: coredns/coredns:1.10.1 command: -conf /root/dnsconfig/Corefile env_file: ./netmaker.env depends_on: From d366c23c63dcfd7957ee766d9a961262b0c7ee6d Mon Sep 17 00:00:00 2001 From: Matthew R Kasun Date: Tue, 22 Aug 2023 02:20:28 -0400 Subject: [PATCH 08/11] validate dns entry does not contain whitespace (#2512) --- controllers/dns_test.go | 13 +++++++++++++ logic/dns.go | 11 +++++++++++ models/dnsEntry.go | 8 ++++---- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/controllers/dns_test.go b/controllers/dns_test.go index e070cfed4..7d2892db0 100644 --- a/controllers/dns_test.go +++ b/controllers/dns_test.go @@ -400,6 +400,19 @@ func TestValidateDNSCreate(t *testing.T) { assert.NotNil(t, err) assert.Contains(t, err.Error(), "Field validation for 'Name' failed on the 'name_unique' tag") }) + t.Run("WhiteSpace", func(t *testing.T) { + entry := models.DNSEntry{Address: "10.10.10.5", Name: "white space", Network: "skynet"} + err := logic.ValidateDNSCreate(entry) + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "Field validation for 'Name' failed on the 'whitespace' tag") + }) + t.Run("AllSpaces", func(t *testing.T) { + entry := models.DNSEntry{Address: "10.10.10.5", Name: " ", Network: "skynet"} + err := logic.ValidateDNSCreate(entry) + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "Field validation for 'Name' failed on the 'whitespace' tag") + }) + } func createHost() { diff --git a/logic/dns.go b/logic/dns.go index 0991c8ce7..15c25deff 100644 --- a/logic/dns.go +++ b/logic/dns.go @@ -3,6 +3,7 @@ package logic import ( "encoding/json" "os" + "regexp" "sort" validator "github.com/go-playground/validator/v10" @@ -203,6 +204,11 @@ func ValidateDNSCreate(entry models.DNSEntry) error { v := validator.New() + _ = v.RegisterValidation("whitespace", func(f1 validator.FieldLevel) bool { + match, _ := regexp.MatchString(`\s`, entry.Name) + return !match + }) + _ = v.RegisterValidation("name_unique", func(fl validator.FieldLevel) bool { num, err := GetDNSEntryNum(entry.Name, entry.Network) return err == nil && num == 0 @@ -227,6 +233,11 @@ func ValidateDNSUpdate(change models.DNSEntry, entry models.DNSEntry) error { v := validator.New() + _ = v.RegisterValidation("whitespace", func(f1 validator.FieldLevel) bool { + match, _ := regexp.MatchString(`\s`, entry.Name) + return !match + }) + _ = v.RegisterValidation("name_unique", func(fl validator.FieldLevel) bool { //if name & net not changing name we are good if change.Name == entry.Name && change.Network == entry.Network { diff --git a/models/dnsEntry.go b/models/dnsEntry.go index 73373f1a1..11e9dd6b4 100644 --- a/models/dnsEntry.go +++ b/models/dnsEntry.go @@ -42,8 +42,8 @@ type DNSUpdate struct { // DNSEntry - a DNS entry represented as struct type DNSEntry struct { - Address string `json:"address" bson:"address" validate:"ip"` - Address6 string `json:"address6" bson:"address6"` - Name string `json:"name" bson:"name" validate:"required,name_unique,min=1,max=192"` - Network string `json:"network" bson:"network" validate:"network_exists"` + Address string `json:"address" validate:"ip"` + Address6 string `json:"address6"` + Name string `json:"name" validate:"required,name_unique,min=1,max=192,whitespace"` + Network string `json:"network" validate:"network_exists"` } From 095617c79f891df66cfc464618873e02b120d9af Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Aug 2023 14:12:18 +0530 Subject: [PATCH 09/11] Bump github.com/google/uuid from 1.3.0 to 1.3.1 (#2530) Bumps [github.com/google/uuid](https://github.com/google/uuid) from 1.3.0 to 1.3.1. - [Release notes](https://github.com/google/uuid/releases) - [Changelog](https://github.com/google/uuid/blob/master/CHANGELOG.md) - [Commits](https://github.com/google/uuid/compare/v1.3.0...v1.3.1) --- updated-dependencies: - dependency-name: github.com/google/uuid dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f73dba38c..b334c074f 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/eclipse/paho.mqtt.golang v1.4.3 github.com/go-playground/validator/v10 v10.15.0 github.com/golang-jwt/jwt/v4 v4.5.0 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.3.1 github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 github.com/lib/pq v1.10.9 diff --git a/go.sum b/go.sum index 537e31453..7405eba6d 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= From bfd98384bb620004d1a299d7873ecb66d7d35c74 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Aug 2023 14:17:47 +0530 Subject: [PATCH 10/11] Bump github.com/go-playground/validator/v10 from 10.15.0 to 10.15.1 (#2529) Bumps [github.com/go-playground/validator/v10](https://github.com/go-playground/validator) from 10.15.0 to 10.15.1. - [Release notes](https://github.com/go-playground/validator/releases) - [Commits](https://github.com/go-playground/validator/compare/v10.15.0...v10.15.1) --- updated-dependencies: - dependency-name: github.com/go-playground/validator/v10 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b334c074f..235fb9cc8 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.19 require ( github.com/eclipse/paho.mqtt.golang v1.4.3 - github.com/go-playground/validator/v10 v10.15.0 + github.com/go-playground/validator/v10 v10.15.1 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/uuid v1.3.1 github.com/gorilla/handlers v1.5.1 diff --git a/go.sum b/go.sum index 7405eba6d..bdcc0c8db 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,8 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.15.0 h1:nDU5XeOKtB3GEa+uB7GNYwhVKsgjAR7VgKoNB6ryXfw= -github.com/go-playground/validator/v10 v10.15.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/go-playground/validator/v10 v10.15.1 h1:BSe8uhN+xQ4r5guV/ywQI4gO59C2raYcGffYWZEjZzM= +github.com/go-playground/validator/v10 v10.15.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= From d2b8daac753546498ab64962528e9ad896a0bb4d Mon Sep 17 00:00:00 2001 From: Matthew R Kasun Date: Wed, 23 Aug 2023 09:53:12 -0400 Subject: [PATCH 11/11] update node expiration during migration (#2532) --- controllers/migrate.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/controllers/migrate.go b/controllers/migrate.go index 6bf2e540f..0d365664c 100644 --- a/controllers/migrate.go +++ b/controllers/migrate.go @@ -5,7 +5,6 @@ import ( "fmt" "net" "net/http" - "strconv" "time" "github.com/google/uuid" @@ -201,7 +200,7 @@ func convertLegacyNode(legacy models.LegacyNode, hostID uuid.UUID) models.Node { node.DNSOn = models.ParseBool(legacy.DNSOn) node.PersistentKeepalive = time.Duration(int64(time.Second) * int64(legacy.PersistentKeepalive)) node.LastModified = time.Now() - node.ExpirationDateTime, _ = time.Parse(strconv.Itoa(int(legacy.ExpirationDateTime)), "0") + node.ExpirationDateTime = time.Unix(legacy.ExpirationDateTime, 0) node.EgressGatewayNatEnabled = models.ParseBool(legacy.EgressGatewayNatEnabled) node.EgressGatewayRequest = legacy.EgressGatewayRequest node.IngressGatewayRange = legacy.IngressGatewayRange