From a686feb8b4b1ffba77492aa087b4b6cb9f6ed9d8 Mon Sep 17 00:00:00 2001 From: Matthew R Kasun Date: Fri, 28 Jul 2023 08:07:49 -0400 Subject: [PATCH 01/33] use NewHosts to init hostfile --- logic/dns.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/logic/dns.go b/logic/dns.go index 0991c8ce7..bee62944f 100644 --- a/logic/dns.go +++ b/logic/dns.go @@ -14,7 +14,10 @@ import ( // SetDNS - sets the dns on file func SetDNS() error { - hostfile := txeh.Hosts{} + hostfile, err := txeh.NewHosts(&txeh.HostsConfig{}) + if err != nil { + return err + } var corefilestring string networks, err := GetNetworks() if err != nil && !database.IsEmptyRecord(err) { From 4402302366bb85f47510dca1ebbdb29d99d307a5 Mon Sep 17 00:00:00 2001 From: Abhishek K <32607604+abhishek9686@users.noreply.github.com> Date: Fri, 28 Jul 2023 19:12:18 +0530 Subject: [PATCH 02/33] Revert "Bump github.com/txn2/txeh from 1.4.0 to 1.5.3 (#2464)" (#2475) This reverts commit aafaa91af0c922157cd49a8dcb059ed2e3d3074f. --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 127a1baae..6a95af3ae 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/rqlite/gorqlite v0.0.0-20210514125552-08ff1e76b22f github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/stretchr/testify v1.8.4 - github.com/txn2/txeh v1.5.3 + github.com/txn2/txeh v1.4.0 golang.org/x/crypto v0.11.0 golang.org/x/net v0.12.0 // indirect golang.org/x/oauth2 v0.10.0 diff --git a/go.sum b/go.sum index 6361884e6..f6bc93af4 100644 --- a/go.sum +++ b/go.sum @@ -109,8 +109,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/txn2/txeh v1.5.3 h1:ZMgc3r+5/AFtE/ayCoICpvxj7xl/CYsZjnIGhozV/Kc= -github.com/txn2/txeh v1.5.3/go.mod h1:qYzGG9kCzeVEI12geK4IlanHWY8X4uy/I3NcW7mk8g4= +github.com/txn2/txeh v1.4.0 h1:0tdvpA4HGJrj8X3kmrU6o/JFStI009nKxwDpMK5CnRU= +github.com/txn2/txeh v1.4.0/go.mod h1:Mgq0hY184zCrDBLgvkIp+9NYGHoYbJcu4xKqUcx1shc= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c h1:3lbZUMbMiGUW/LMkfsEABsc5zNT9+b1CvsJx47JzJ8g= github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c/go.mod h1:UrdRz5enIKZ63MEE3IF9l2/ebyx59GyGgPi+tICQdmM= From 8bf3b931e92681191cd1ff0785755a93a378c085 Mon Sep 17 00:00:00 2001 From: Abhishek K <32607604+abhishek9686@users.noreply.github.com> Date: Fri, 28 Jul 2023 19:44:51 +0530 Subject: [PATCH 03/33] NET-433: Only Set Routes for NAT enabled egress GW (#2469) * remove related fields and code * remover metrics collection from server code * fw update struct * add ext client flag to metrics data * simply nat types * rm proxy update from cli * remove ingress routes from firewall update * check if egress ranges are present * rm unused func --- logic/peers.go | 96 +------------------------------------------------- models/mqtt.go | 15 ++++---- 2 files changed, 7 insertions(+), 104 deletions(-) diff --git a/logic/peers.go b/logic/peers.go index b5a15b24b..9068a4a65 100644 --- a/logic/peers.go +++ b/logic/peers.go @@ -30,9 +30,6 @@ func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.N ServerVersion: servercfg.GetVersion(), ServerAddrs: []models.ServerAddr{}, FwUpdate: models.FwUpdate{ - IngressInfo: models.IngressInfo{ - ExtPeers: make(map[string]models.ExtClientInfo), - }, EgressInfo: make(map[string]models.EgressInfo), }, PeerIDs: make(models.PeerMap, 0), @@ -110,10 +107,6 @@ func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.N } currentPeers := GetNetworkNodesMemory(allNodes, node.Network) - var nodePeerMap map[string]models.PeerRouteInfo - if node.IsIngressGateway || node.IsEgressGateway { - nodePeerMap = make(map[string]models.PeerRouteInfo) - } for _, peer := range currentPeers { peer := peer if peer.ID.String() == node.ID.String() { @@ -138,38 +131,6 @@ func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.N EgressRanges: peer.EgressGatewayRanges, }) } - if node.IsIngressGateway || node.IsEgressGateway { - if peer.IsIngressGateway { - _, extPeerIDAndAddrs, err := getExtPeers(&peer, &node) - if err == nil { - for _, extPeerIdAndAddr := range extPeerIDAndAddrs { - extPeerIdAndAddr := extPeerIdAndAddr - nodePeerMap[extPeerIdAndAddr.ID] = models.PeerRouteInfo{ - PeerAddr: net.IPNet{ - IP: net.ParseIP(extPeerIdAndAddr.Address), - Mask: getCIDRMaskFromAddr(extPeerIdAndAddr.Address), - }, - PeerKey: extPeerIdAndAddr.ID, - Allow: true, - ID: extPeerIdAndAddr.ID, - } - } - } - } - if node.IsIngressGateway && peer.IsEgressGateway { - hostPeerUpdate.FwUpdate.IngressInfo.EgressRanges = append(hostPeerUpdate.FwUpdate.IngressInfo.EgressRanges, - peer.EgressGatewayRanges...) - } - nodePeerMap[peerHost.PublicKey.String()] = models.PeerRouteInfo{ - PeerAddr: net.IPNet{ - IP: net.ParseIP(peer.PrimaryAddress()), - Mask: getCIDRMaskFromAddr(peer.PrimaryAddress()), - }, - PeerKey: peerHost.PublicKey.String(), - Allow: true, - ID: peer.ID.String(), - } - } if (node.IsRelayed && node.RelayedBy != peer.ID.String()) || (peer.IsRelayed && peer.RelayedBy != node.ID.String()) { // if node is relayed and peer is not the relay, set remove to true if _, ok := peerIndexMap[peerHost.PublicKey.String()]; ok { @@ -248,39 +209,11 @@ func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.N var extPeers []wgtypes.PeerConfig var extPeerIDAndAddrs []models.IDandAddr if node.IsIngressGateway { - hostPeerUpdate.FwUpdate.IsIngressGw = true extPeers, extPeerIDAndAddrs, err = getExtPeers(&node, &node) if err == nil { - for _, extPeerIdAndAddr := range extPeerIDAndAddrs { - extPeerIdAndAddr := extPeerIdAndAddr - nodePeerMap[extPeerIdAndAddr.ID] = models.PeerRouteInfo{ - PeerAddr: net.IPNet{ - IP: net.ParseIP(extPeerIdAndAddr.Address), - Mask: getCIDRMaskFromAddr(extPeerIdAndAddr.Address), - }, - PeerKey: extPeerIdAndAddr.ID, - Allow: true, - ID: extPeerIdAndAddr.ID, - } - } hostPeerUpdate.Peers = append(hostPeerUpdate.Peers, extPeers...) for _, extPeerIdAndAddr := range extPeerIDAndAddrs { extPeerIdAndAddr := extPeerIdAndAddr - - hostPeerUpdate.FwUpdate.IngressInfo.ExtPeers[extPeerIdAndAddr.ID] = models.ExtClientInfo{ - Masquerade: true, - IngGwAddr: net.IPNet{ - IP: net.ParseIP(node.PrimaryAddress()), - Mask: getCIDRMaskFromAddr(node.PrimaryAddress()), - }, - Network: node.PrimaryNetworkRange(), - ExtPeerAddr: net.IPNet{ - IP: net.ParseIP(extPeerIdAndAddr.Address), - Mask: getCIDRMaskFromAddr(extPeerIdAndAddr.Address), - }, - ExtPeerKey: extPeerIdAndAddr.ID, - Peers: filterNodeMapForClientACLs(extPeerIdAndAddr.ID, node.Network, nodePeerMap), - } if node.Network == network { hostPeerUpdate.PeerIDs[extPeerIdAndAddr.ID] = extPeerIdAndAddr hostPeerUpdate.NodePeers = append(hostPeerUpdate.NodePeers, extPeers...) @@ -290,7 +223,7 @@ func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.N logger.Log(1, "error retrieving external clients:", err.Error()) } } - if node.IsEgressGateway { + if node.IsEgressGateway && node.EgressGatewayRequest.NatEnabled == "yes" && len(node.EgressGatewayRequest.Ranges) > 0 { hostPeerUpdate.FwUpdate.IsEgressGw = true hostPeerUpdate.FwUpdate.EgressInfo[node.ID.String()] = models.EgressInfo{ EgressID: node.ID.String(), @@ -299,7 +232,6 @@ func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.N IP: net.ParseIP(node.PrimaryAddress()), Mask: getCIDRMaskFromAddr(node.PrimaryAddress()), }, - GwPeers: nodePeerMap, EgressGWCfg: node.EgressGatewayRequest, } } @@ -584,29 +516,3 @@ func getCIDRMaskFromAddr(addr string) net.IPMask { } return cidr } - -// accounts for ext client ACLs -func filterNodeMapForClientACLs(publicKey, network string, nodePeerMap map[string]models.PeerRouteInfo) map[string]models.PeerRouteInfo { - if !isEE { - return nodePeerMap - } - if nodePeerMap == nil { - return map[string]models.PeerRouteInfo{} - } - - if len(publicKey) == 0 || len(network) == 0 { - return nodePeerMap - } - - client, err := GetExtClientByPubKey(publicKey, network) - if err != nil { - return nodePeerMap - } - for k := range nodePeerMap { - currNodePeer := nodePeerMap[k] - if _, ok := client.DeniedACLs[currNodePeer.ID]; ok { - delete(nodePeerMap, k) - } - } - return nodePeerMap -} diff --git a/models/mqtt.go b/models/mqtt.go index e818df674..b50e8cd22 100644 --- a/models/mqtt.go +++ b/models/mqtt.go @@ -30,11 +30,10 @@ type IngressInfo struct { // EgressInfo - struct for egress info type EgressInfo struct { - EgressID string `json:"egress_id" yaml:"egress_id"` - Network net.IPNet `json:"network" yaml:"network"` - EgressGwAddr net.IPNet `json:"egress_gw_addr" yaml:"egress_gw_addr"` - GwPeers map[string]PeerRouteInfo `json:"gateway_peers" yaml:"gateway_peers"` - EgressGWCfg EgressGatewayRequest `json:"egress_gateway_cfg" yaml:"egress_gateway_cfg"` + EgressID string `json:"egress_id" yaml:"egress_id"` + Network net.IPNet `json:"network" yaml:"network"` + EgressGwAddr net.IPNet `json:"egress_gw_addr" yaml:"egress_gw_addr"` + EgressGWCfg EgressGatewayRequest `json:"egress_gateway_cfg" yaml:"egress_gateway_cfg"` } // EgressNetworkRoutes - struct for egress network routes for adding routes to peer's interface @@ -69,8 +68,6 @@ type KeyUpdate struct { // FwUpdate - struct for firewall updates type FwUpdate struct { - IsIngressGw bool `json:"is_ingress_gw"` - IsEgressGw bool `json:"is_egress_gw"` - IngressInfo IngressInfo `json:"ingress_info"` - EgressInfo map[string]EgressInfo `json:"egress_info"` + IsEgressGw bool `json:"is_egress_gw"` + EgressInfo map[string]EgressInfo `json:"egress_info"` } From bb8e150ad3be89b1fcfec960c41f6011fa16a2d3 Mon Sep 17 00:00:00 2001 From: Matthew R Kasun Date: Fri, 28 Jul 2023 11:46:35 -0400 Subject: [PATCH 04/33] ignore network parameter passed to delete ingress endpoint (#2471) --- controllers/node.go | 2 +- go.mod | 6 ------ go.sum | 8 -------- logic/gateway.go | 8 ++++---- 4 files changed, 5 insertions(+), 19 deletions(-) diff --git a/controllers/node.go b/controllers/node.go index 1d92acd58..3b4aceed6 100644 --- a/controllers/node.go +++ b/controllers/node.go @@ -566,7 +566,7 @@ func deleteIngressGateway(w http.ResponseWriter, r *http.Request) { var params = mux.Vars(r) nodeid := params["nodeid"] netid := params["network"] - node, wasFailover, removedClients, err := logic.DeleteIngressGateway(netid, nodeid) + node, wasFailover, removedClients, err := logic.DeleteIngressGateway(nodeid) if err != nil { logger.Log(0, r.Header.Get("user"), fmt.Sprintf("failed to delete ingress gateway on node [%s] on network [%s]: %v", diff --git a/go.mod b/go.mod index 6a95af3ae..4c5524d08 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,6 @@ require ( golang.org/x/oauth2 v0.10.0 golang.org/x/sys v0.10.0 // indirect golang.org/x/text v0.11.0 // indirect - golang.zx2c4.com/wireguard v0.0.0-20220920152132-bb719d3a6e2c // indirect golang.zx2c4.com/wireguard/wgctrl v0.0.0-20220324164955-056925b7df31 google.golang.org/protobuf v1.31.0 // indirect gopkg.in/yaml.v3 v3.0.1 @@ -62,14 +61,9 @@ require ( github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/go-cmp v0.5.9 // indirect github.com/hashicorp/go-version v1.6.0 - github.com/josharian/native v1.0.0 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect - github.com/mdlayher/genetlink v1.2.0 // indirect - github.com/mdlayher/netlink v1.6.0 // indirect - github.com/mdlayher/socket v0.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect golang.org/x/sync v0.1.0 // indirect diff --git a/go.sum b/go.sum index f6bc93af4..d71d2c468 100644 --- a/go.sum +++ b/go.sum @@ -43,7 +43,6 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= @@ -58,7 +57,6 @@ github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mO github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/josharian/native v1.0.0 h1:Ts/E8zCSEsG17dUqv7joXJFybuMLjQfWE04tsBODTxk= github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= @@ -72,13 +70,9 @@ github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/mdlayher/genetlink v1.2.0 h1:4yrIkRV5Wfk1WfpWTcoOlGmsWgQj3OtQN9ZsbrE+XtU= github.com/mdlayher/genetlink v1.2.0/go.mod h1:ra5LDov2KrUCZJiAtEvXXZBxGMInICMXIwshlJ+qRxQ= -github.com/mdlayher/netlink v1.6.0 h1:rOHX5yl7qnlpiVkFWoqccueppMtXzeziFjWAjLg6sz0= github.com/mdlayher/netlink v1.6.0/go.mod h1:0o3PlBmGst1xve7wQ7j/hwpNaFaH4qCRyWCdcZk8/vA= -github.com/mdlayher/socket v0.1.1 h1:q3uOGirUPfAV2MUoaC7BavjQ154J7+JOkTWyiV+intI= github.com/mdlayher/socket v0.1.1/go.mod h1:mYV5YIZAfHh4dzDVzI8x8tWLWCliuX8Mon5Awbj+qDs= -github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721 h1:RlZweED6sbSArvlE924+mUcZuXKLBHA35U7LN621Bws= github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721/go.mod h1:Ickgr2WtCLZ2MDGd4Gr0geeCH5HybhRJbonOgQpvSxc= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= @@ -164,8 +158,6 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T golang.zx2c4.com/go118/netip v0.0.0-20211111135330-a4a02eeacf9d/go.mod h1:5yyfuiqVIJ7t+3MqrpTQ+QqRkMWiESiyDvPNvKYCecg= golang.zx2c4.com/wintun v0.0.0-20211104114900-415007cec224/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI= golang.zx2c4.com/wireguard v0.0.0-20220202223031-3b95c81cc178/go.mod h1:TjUWrnD5ATh7bFvmm/ALEJZQ4ivKbETb6pmyj1vUoNI= -golang.zx2c4.com/wireguard v0.0.0-20220920152132-bb719d3a6e2c h1:Okh6a1xpnJslG9Mn84pId1Mn+Q8cvpo4HCeeFWHo0cA= -golang.zx2c4.com/wireguard v0.0.0-20220920152132-bb719d3a6e2c/go.mod h1:enML0deDxY1ux+B6ANGiwtg0yAJi1rctkTpcHNAVPyg= golang.zx2c4.com/wireguard/wgctrl v0.0.0-20220324164955-056925b7df31 h1:AgW3hljgTzuRbCB0j+q9tXT0uy6ij7vMjEzSCeMlQY0= golang.zx2c4.com/wireguard/wgctrl v0.0.0-20220324164955-056925b7df31/go.mod h1:8P32Ilp1kCpwB4ItaHyvSk4xAtnpQ+8gQVfg5WaO1TU= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= diff --git a/logic/gateway.go b/logic/gateway.go index 083092ee5..90827e13f 100644 --- a/logic/gateway.go +++ b/logic/gateway.go @@ -127,13 +127,13 @@ func CreateIngressGateway(netid string, nodeid string, ingress models.IngressReq } // DeleteIngressGateway - deletes an ingress gateway -func DeleteIngressGateway(networkName string, nodeid string) (models.Node, bool, []models.ExtClient, error) { +func DeleteIngressGateway(nodeid string) (models.Node, bool, []models.ExtClient, error) { removedClients := []models.ExtClient{} node, err := GetNodeByID(nodeid) if err != nil { return models.Node{}, false, removedClients, err } - clients, err := GetExtClientsByID(nodeid, networkName) + clients, err := GetExtClientsByID(nodeid, node.Network) if err != nil && !database.IsEmptyRecord(err) { return models.Node{}, false, removedClients, err } @@ -141,7 +141,7 @@ func DeleteIngressGateway(networkName string, nodeid string) (models.Node, bool, removedClients = clients // delete ext clients belonging to ingress gateway - if err = DeleteGatewayExtClients(node.ID.String(), networkName); err != nil { + if err = DeleteGatewayExtClients(node.ID.String(), node.Network); err != nil { return models.Node{}, false, removedClients, err } logger.Log(3, "deleting ingress gateway") @@ -163,7 +163,7 @@ func DeleteIngressGateway(networkName string, nodeid string) (models.Node, bool, if err != nil { return models.Node{}, wasFailover, removedClients, err } - err = SetNetworkNodesLastModified(networkName) + err = SetNetworkNodesLastModified(node.Network) return node, wasFailover, removedClients, err } From b8f387a0ac4dfdfdc3b69c748fee6caf3a0b1a31 Mon Sep 17 00:00:00 2001 From: Aceix Date: Fri, 28 Jul 2023 15:58:36 +0000 Subject: [PATCH 05/33] fix(NET-388): fix to update node dns toggle (#2472) --- models/node.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/models/node.go b/models/node.go index f9c3867bb..ba236cbea 100644 --- a/models/node.go +++ b/models/node.go @@ -358,7 +358,7 @@ func (node *LegacyNode) SetDefaultFailover() { } } -// Node.Fill - fills other node data into calling node data if not set on calling node +// Node.Fill - fills other node data into calling node data if not set on calling node (skips DNSOn) func (newNode *Node) Fill(currentNode *Node, isEE bool) { // TODO add new field for nftables present newNode.ID = currentNode.ID newNode.HostID = currentNode.HostID @@ -404,9 +404,6 @@ func (newNode *Node) Fill(currentNode *Node, isEE bool) { // TODO add new field if newNode.IngressGatewayRange6 == "" { newNode.IngressGatewayRange6 = currentNode.IngressGatewayRange6 } - if newNode.DNSOn != currentNode.DNSOn { - newNode.DNSOn = currentNode.DNSOn - } if newNode.Action == "" { newNode.Action = currentNode.Action } From 5fd805a148f5d9a7cced9498dd00dc5d278c17cb Mon Sep 17 00:00:00 2001 From: Abhishek K <32607604+abhishek9686@users.noreply.github.com> Date: Mon, 31 Jul 2023 19:02:22 +0530 Subject: [PATCH 06/33] use local port for metrics pinger (#2479) --- logic/peers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logic/peers.go b/logic/peers.go index 9068a4a65..258a505f7 100644 --- a/logic/peers.go +++ b/logic/peers.go @@ -201,7 +201,7 @@ func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.N Address: peer.PrimaryAddress(), Name: peerHost.Name, Network: peer.Network, - ListenPort: GetPeerListenPort(peerHost), + ListenPort: peerHost.ListenPort, } hostPeerUpdate.NodePeers = append(hostPeerUpdate.NodePeers, nodePeer) } From 878d0a9d5f8cfdff2f9c7ea6893df7184e5c9eb1 Mon Sep 17 00:00:00 2001 From: Abhishek K <32607604+abhishek9686@users.noreply.github.com> Date: Tue, 1 Aug 2023 01:53:24 +0530 Subject: [PATCH 07/33] v0.20.5 release notes (#2482) * v0.20.5 release notes * add a known issue --- release.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/release.md b/release.md index df6d1bbd5..3367c5e4d 100644 --- a/release.md +++ b/release.md @@ -2,18 +2,18 @@ # Netmaker v0.20.5 ## Whats New -- FreeBSD 13/14 specific binaries -- Whitelabelling capabilities +- Extclient Acls +- Force delete host along with all the associated nodes ## What's Fixed -- Fixes for FreeBSD -- Mac installer installs WireGuard -- ACL rendering on UI -- Updating Endpoint IP from UI +- Deprecated Proxy +- Solved Race condition for multiple nodes joining network at same time +- Node dns toggle +- Simplified Firewall rules for added stability ## known issues +- Expired nodes are not getting cleaned up - Windows installer does not install WireGuard -- netclient-gui (windows) will display an erroneous error dialog when joining a network (can be ignored) - netclient-gui will continously display error dialog if netmaker server is offline - Incorrect metrics against ext clients - Host ListenPorts set to 0 after migration from 0.17.1 -> 0.20.5 From 495f7216d68dd4bd9dcd33355c6b61bdf1e3cec8 Mon Sep 17 00:00:00 2001 From: Christopher Blaha Date: Tue, 1 Aug 2023 02:15:17 -0400 Subject: [PATCH 08/33] update develop to v0.20.6 (#2485) --- .github/ISSUE_TEMPLATE/bug-report.yml | 1 + README.md | 2 +- compose/docker-compose.netclient.yml | 2 +- controllers/docs.go | 2 +- k8s/client/netclient-daemonset.yaml | 2 +- k8s/client/netclient.yaml | 2 +- k8s/server/netmaker-ui.yaml | 2 +- main.go | 2 +- release.md | 4 ++-- scripts/nm-upgrade-0-17-1-to-0-19-0.sh | 2 +- swagger.yaml | 2 +- 11 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index 21ee06169..02c2b48c7 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -31,6 +31,7 @@ body: label: Version description: What version are you running? options: + - v0.20.6 - v0.20.5 - v0.20.4 - v0.20.3 diff --git a/README.md b/README.md index cdc995b76..840ceb301 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@

- + diff --git a/compose/docker-compose.netclient.yml b/compose/docker-compose.netclient.yml index f7f0fe1f6..9144b76cd 100644 --- a/compose/docker-compose.netclient.yml +++ b/compose/docker-compose.netclient.yml @@ -3,7 +3,7 @@ version: "3.4" services: netclient: container_name: netclient - image: 'gravitl/netclient:v0.20.5' + image: 'gravitl/netclient:v0.20.6' hostname: netmaker-1 network_mode: host restart: on-failure diff --git a/controllers/docs.go b/controllers/docs.go index 1e409c47d..592922982 100644 --- a/controllers/docs.go +++ b/controllers/docs.go @@ -10,7 +10,7 @@ // // Schemes: https // BasePath: / -// Version: 0.20.5 +// Version: 0.20.6 // Host: netmaker.io // // Consumes: diff --git a/k8s/client/netclient-daemonset.yaml b/k8s/client/netclient-daemonset.yaml index 57efaa858..635e25769 100644 --- a/k8s/client/netclient-daemonset.yaml +++ b/k8s/client/netclient-daemonset.yaml @@ -16,7 +16,7 @@ spec: hostNetwork: true containers: - name: netclient - image: gravitl/netclient:v0.20.5 + image: gravitl/netclient:v0.20.6 env: - name: TOKEN value: "TOKEN_VALUE" diff --git a/k8s/client/netclient.yaml b/k8s/client/netclient.yaml index 8ab17867c..12a6ea116 100644 --- a/k8s/client/netclient.yaml +++ b/k8s/client/netclient.yaml @@ -28,7 +28,7 @@ spec: # - "" containers: - name: netclient - image: gravitl/netclient:v0.20.5 + image: gravitl/netclient:v0.20.6 env: - name: TOKEN value: "TOKEN_VALUE" diff --git a/k8s/server/netmaker-ui.yaml b/k8s/server/netmaker-ui.yaml index 0037a960d..2d7e62ca5 100644 --- a/k8s/server/netmaker-ui.yaml +++ b/k8s/server/netmaker-ui.yaml @@ -15,7 +15,7 @@ spec: spec: containers: - name: netmaker-ui - image: gravitl/netmaker-ui:v0.20.5 + image: gravitl/netmaker-ui:v0.20.6 ports: - containerPort: 443 env: diff --git a/main.go b/main.go index 6abd97680..0bf365e5a 100644 --- a/main.go +++ b/main.go @@ -29,7 +29,7 @@ import ( "golang.org/x/exp/slog" ) -var version = "v0.20.5" +var version = "v0.20.6" // Start DB Connection and start API Request Handler func main() { diff --git a/release.md b/release.md index 3367c5e4d..82f009f55 100644 --- a/release.md +++ b/release.md @@ -1,5 +1,5 @@ -# Netmaker v0.20.5 +# Netmaker v0.20.6 ## Whats New - Extclient Acls @@ -16,7 +16,7 @@ - Windows installer does not install WireGuard - netclient-gui will continously display error dialog if netmaker server is offline - Incorrect metrics against ext clients -- Host ListenPorts set to 0 after migration from 0.17.1 -> 0.20.5 +- Host ListenPorts set to 0 after migration from 0.17.1 -> 0.20.6 - Mac IPv6 addresses/route issues - Docker client can not re-join after complete deletion - netclient-gui network tab blank after disconnect diff --git a/scripts/nm-upgrade-0-17-1-to-0-19-0.sh b/scripts/nm-upgrade-0-17-1-to-0-19-0.sh index 98139d7d4..359457441 100644 --- a/scripts/nm-upgrade-0-17-1-to-0-19-0.sh +++ b/scripts/nm-upgrade-0-17-1-to-0-19-0.sh @@ -1,6 +1,6 @@ #!/bin/bash -LATEST="v0.20.5" +LATEST="v0.20.6" INSTALL_PATH="/root" trap restore_old_netmaker_instructions diff --git a/swagger.yaml b/swagger.yaml index 392776c08..c71d63877 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -704,7 +704,7 @@ info: API calls must be authenticated via a header of the format -H “Authorization: Bearer ” There are two methods to obtain YOUR_SECRET_KEY: 1. Using the masterkey. By default, this value is “secret key,” but you should change this on your instance and keep it secure. This value can be set via env var at startup or in a config file (config/environments/< env >.yaml). See the [Netmaker](https://docs.netmaker.org/index.html) documentation for more details. 2. Using a JWT received for a node. This can be retrieved by calling the /api/nodes//authenticate endpoint, as documented below. title: Netmaker - version: 0.20.5 + version: 0.20.6 paths: /api/dns: get: From 2ad4653847320f883aa11a9deb39a5c23b2c6494 Mon Sep 17 00:00:00 2001 From: Matthew R Kasun Date: Tue, 1 Aug 2023 10:50:43 -0400 Subject: [PATCH 09/33] enforce unique names for ext client names (#2476) * enforce unique names for ext client names * only check for unique id on creation * check for unique id if changed --- controllers/ext_client.go | 128 ++++++++++++++++++++------------------ controllers/regex.go | 1 + logic/clients.go | 15 +++++ logic/extpeers.go | 14 ++--- 4 files changed, 90 insertions(+), 68 deletions(-) diff --git a/controllers/ext_client.go b/controllers/ext_client.go index ad99ef3b5..4155c6537 100644 --- a/controllers/ext_client.go +++ b/controllers/ext_client.go @@ -17,6 +17,7 @@ import ( "github.com/gravitl/netmaker/models/promodels" "github.com/gravitl/netmaker/mq" "github.com/skip2/go-qrcode" + "golang.org/x/exp/slog" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" ) @@ -308,31 +309,28 @@ func createExtClient(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") var params = mux.Vars(r) - networkName := params["network"] nodeid := params["nodeid"] ingressExists := checkIngressExists(nodeid) if !ingressExists { err := errors.New("ingress does not exist") - logger.Log(0, r.Header.Get("user"), - fmt.Sprintf("failed to create extclient on network [%s]: %v", networkName, err)) - logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal")) + slog.Error("failed to create extclient", "user", r.Header.Get("user"), "error", err) + logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest")) return } - var extclient models.ExtClient var customExtClient models.CustomExtClient if err := json.NewDecoder(r.Body).Decode(&customExtClient); err != nil { logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest")) return } - if err := validateExtClient(&extclient, &customExtClient); err != nil { + if err := validateCustomExtClient(&customExtClient, true); err != nil { logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest")) return } + extclient := logic.UpdateExtClient(&models.ExtClient{}, &customExtClient) - extclient.Network = networkName extclient.IngressGatewayID = nodeid node, err := logic.GetNodeByID(nodeid) if err != nil { @@ -341,6 +339,7 @@ func createExtClient(w http.ResponseWriter, r *http.Request) { logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal")) return } + extclient.Network = node.Network host, err := logic.GetHost(node.HostID.String()) if err != nil { logger.Log(0, r.Header.Get("user"), @@ -351,21 +350,19 @@ func createExtClient(w http.ResponseWriter, r *http.Request) { listenPort := logic.GetPeerListenPort(host) extclient.IngressGatewayEndpoint = fmt.Sprintf("%s:%d", host.EndpointIP.String(), listenPort) extclient.Enabled = true - parentNetwork, err := logic.GetNetwork(networkName) + parentNetwork, err := logic.GetNetwork(node.Network) if err == nil { // check if parent network default ACL is enabled (yes) or not (no) extclient.Enabled = parentNetwork.DefaultACL == "yes" } if err := logic.SetClientDefaultACLs(&extclient); err != nil { - logger.Log(0, r.Header.Get("user"), - fmt.Sprintf("failed to assign ACLs to new ext client on network [%s]: %v", networkName, err)) + slog.Error("failed to set default acls for extclient", "user", r.Header.Get("user"), "network", node.Network, "error", err) logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal")) return } if err = logic.CreateExtClient(&extclient); err != nil { - logger.Log(0, r.Header.Get("user"), - fmt.Sprintf("failed to create new ext client on network [%s]: %v", networkName, err)) + slog.Error("failed to create extclient", "user", r.Header.Get("user"), "network", node.Network, "error", err) logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal")) return } @@ -374,13 +371,13 @@ func createExtClient(w http.ResponseWriter, r *http.Request) { if r.Header.Get("ismaster") != "yes" { userID := r.Header.Get("user") if isAdmin, err = checkProClientAccess(userID, extclient.ClientID, &parentNetwork); err != nil { - logger.Log(0, userID, "attempted to create a client on network", networkName, "but they lack access") - logic.DeleteExtClient(networkName, extclient.ClientID) + slog.Error("pro client access check failed", "user", userID, "network", node.Network, "error", err) + logic.DeleteExtClient(node.Network, extclient.ClientID) logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal")) return } if !isAdmin { - if err = pro.AssociateNetworkUserClient(userID, networkName, extclient.ClientID); err != nil { + if err = pro.AssociateNetworkUserClient(userID, node.Network, extclient.ClientID); err != nil { logger.Log(0, "failed to associate client", extclient.ClientID, "to user", userID) } extclient.OwnerID = userID @@ -390,7 +387,7 @@ func createExtClient(w http.ResponseWriter, r *http.Request) { } } - logger.Log(0, r.Header.Get("user"), "created new ext client on network", networkName) + slog.Info("created extclient", "user", r.Header.Get("user"), "network", node.Network, "clientid", extclient.ClientID) w.WriteHeader(http.StatusOK) go func() { if err := mq.PublishPeerUpdate(); err != nil { @@ -419,7 +416,7 @@ func updateExtClient(w http.ResponseWriter, r *http.Request) { var params = mux.Vars(r) var update models.CustomExtClient - var oldExtClient models.ExtClient + //var oldExtClient models.ExtClient var sendPeerUpdate bool err := json.NewDecoder(r.Body).Decode(&update) if err != nil { @@ -429,50 +426,40 @@ func updateExtClient(w http.ResponseWriter, r *http.Request) { return } clientid := params["clientid"] - network := params["network"] - key, err := logic.GetRecordKey(clientid, network) + oldExtClient, err := logic.GetExtClientByName(clientid) if err != nil { - logger.Log(0, r.Header.Get("user"), - fmt.Sprintf("failed to get record key for client [%s], network [%s]: %v", - clientid, network, err)) + slog.Error("failed to retrieve extclient", "user", r.Header.Get("user"), "id", clientid, "error", err) logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal")) return } - if err := validateExtClient(&oldExtClient, &update); err != nil { - logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest")) - return - } - data, err := database.FetchRecord(database.EXT_CLIENT_TABLE_NAME, key) - if err != nil { - logger.Log(0, r.Header.Get("user"), - fmt.Sprintf("failed to fetch ext client record key [%s] from db for client [%s], network [%s]: %v", - key, clientid, network, err)) - logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal")) - return - } - if err = json.Unmarshal([]byte(data), &oldExtClient); err != nil { - logger.Log(0, "error unmarshalling extclient: ", - err.Error()) - logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal")) - return + if oldExtClient.ClientID == update.ClientID { + if err := validateCustomExtClient(&update, false); err != nil { + logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest")) + return + } + } else { + if err := validateCustomExtClient(&update, true); err != nil { + logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest")) + return + } } // == PRO == - networkName := params["network"] + //networkName := params["network"] var changedID = update.ClientID != oldExtClient.ClientID if r.Header.Get("ismaster") != "yes" { userID := r.Header.Get("user") - _, doesOwn := doesUserOwnClient(userID, params["clientid"], networkName) + _, doesOwn := doesUserOwnClient(userID, params["clientid"], oldExtClient.Network) if !doesOwn { logic.ReturnErrorResponse(w, r, logic.FormatError(fmt.Errorf("user not permitted"), "internal")) return } } if changedID && oldExtClient.OwnerID != "" { - if err := pro.DissociateNetworkUserClient(oldExtClient.OwnerID, networkName, oldExtClient.ClientID); err != nil { + if err := pro.DissociateNetworkUserClient(oldExtClient.OwnerID, oldExtClient.Network, oldExtClient.ClientID); err != nil { logger.Log(0, "failed to dissociate client", oldExtClient.ClientID, "from user", oldExtClient.OwnerID) } - if err := pro.AssociateNetworkUserClient(oldExtClient.OwnerID, networkName, update.ClientID); err != nil { + if err := pro.AssociateNetworkUserClient(oldExtClient.OwnerID, oldExtClient.Network, update.ClientID); err != nil { logger.Log(0, "failed to associate client", update.ClientID, "to user", oldExtClient.OwnerID) } } @@ -485,13 +472,15 @@ func updateExtClient(w http.ResponseWriter, r *http.Request) { if update.Enabled != oldExtClient.Enabled { sendPeerUpdate = true } - // extra var need as logic.Update changes oldExtClient - currentClient := oldExtClient - newclient, err := logic.UpdateExtClient(&oldExtClient, &update) - if err != nil { - logger.Log(0, r.Header.Get("user"), - fmt.Sprintf("failed to update ext client [%s], network [%s]: %v", - clientid, network, err)) + newclient := logic.UpdateExtClient(&oldExtClient, &update) + if err := logic.DeleteExtClient(oldExtClient.Network, oldExtClient.ClientID); err != nil { + + slog.Error("failed to delete ext client", "user", r.Header.Get("user"), "id", oldExtClient.ClientID, "network", oldExtClient.Network, "error", err) + logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal")) + return + } + if err := logic.SaveExtClient(&newclient); err != nil { + slog.Error("failed to save ext client", "user", r.Header.Get("user"), "id", newclient.ClientID, "network", newclient.Network, "error", err) logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal")) return } @@ -507,7 +496,7 @@ func updateExtClient(w http.ResponseWriter, r *http.Request) { json.NewEncoder(w).Encode(newclient) if changedID { go func() { - if err := mq.PublishExtClientDNSUpdate(currentClient, *newclient, networkName); err != nil { + if err := mq.PublishExtClientDNSUpdate(oldExtClient, newclient, oldExtClient.Network); err != nil { logger.Log(1, "error pubishing dns update for extcient update", err.Error()) } }() @@ -647,18 +636,20 @@ func doesUserOwnClient(username, clientID, network string) (bool, bool) { return false, logic.StringSliceContains(netUser.Clients, clientID) } -// validateExtClient Validates the extclient object -func validateExtClient(extclient *models.ExtClient, customExtClient *models.CustomExtClient) error { +// validateCustomExtClient Validates the extclient object +func validateCustomExtClient(customExtClient *models.CustomExtClient, checkID bool) error { //validate clientid - if customExtClient.ClientID != "" && !validName(customExtClient.ClientID) { - return errInvalidExtClientID + if customExtClient.ClientID != "" { + if err := isValid(customExtClient.ClientID, checkID); err != nil { + return fmt.Errorf("client validatation: %v", err) + } } - extclient.ClientID = customExtClient.ClientID + //extclient.ClientID = customExtClient.ClientID if len(customExtClient.PublicKey) > 0 { if _, err := wgtypes.ParseKey(customExtClient.PublicKey); err != nil { return errInvalidExtClientPubKey } - extclient.PublicKey = customExtClient.PublicKey + //extclient.PublicKey = customExtClient.PublicKey } //validate extra ips if len(customExtClient.ExtraAllowedIPs) > 0 { @@ -667,14 +658,33 @@ func validateExtClient(extclient *models.ExtClient, customExtClient *models.Cust return errInvalidExtClientExtraIP } } - extclient.ExtraAllowedIPs = customExtClient.ExtraAllowedIPs + //extclient.ExtraAllowedIPs = customExtClient.ExtraAllowedIPs } //validate DNS if customExtClient.DNS != "" { if ip := net.ParseIP(customExtClient.DNS); ip == nil { return errInvalidExtClientDNS } - extclient.DNS = customExtClient.DNS + //extclient.DNS = customExtClient.DNS + } + return nil +} + +// isValid Checks if the clientid is valid +func isValid(clientid string, checkID bool) error { + if !validName(clientid) { + return errInvalidExtClientID + } + if checkID { + extclients, err := logic.GetAllExtClients() + if err != nil { + return fmt.Errorf("extclients isValid: %v", err) + } + for _, extclient := range extclients { + if clientid == extclient.ClientID { + return errDuplicateExtClientName + } + } } return nil } diff --git a/controllers/regex.go b/controllers/regex.go index 668c59845..ae913323c 100644 --- a/controllers/regex.go +++ b/controllers/regex.go @@ -10,6 +10,7 @@ var ( errInvalidExtClientID = errors.New("ext client ID must be alphanumderic and/or dashes and less that 15 chars") errInvalidExtClientExtraIP = errors.New("ext client extra ip must be a valid cidr") errInvalidExtClientDNS = errors.New("ext client dns must be a valid ip address") + errDuplicateExtClientName = errors.New("duplicate client name") ) // allow only dashes and alphaneumeric for ext client and node names diff --git a/logic/clients.go b/logic/clients.go index 685a128bb..db6dbab67 100644 --- a/logic/clients.go +++ b/logic/clients.go @@ -1,6 +1,7 @@ package logic import ( + "errors" "sort" "github.com/gravitl/netmaker/models" @@ -70,3 +71,17 @@ func SortExtClient(unsortedExtClient []models.ExtClient) { return unsortedExtClient[i].ClientID < unsortedExtClient[j].ClientID }) } + +// GetExtClientByName - gets an ext client by name +func GetExtClientByName(ID string) (models.ExtClient, error) { + clients, err := GetAllExtClients() + if err != nil { + return models.ExtClient{}, err + } + for i := range clients { + if clients[i].ClientID == ID { + return clients[i], nil + } + } + return models.ExtClient{}, errors.New("client not found") +} diff --git a/logic/extpeers.go b/logic/extpeers.go index 6b6e1a716..7e3fb2fbe 100644 --- a/logic/extpeers.go +++ b/logic/extpeers.go @@ -152,9 +152,9 @@ func GetExtClientByPubKey(publicKey string, network string) (*models.ExtClient, return nil, fmt.Errorf("no client found") } -// CreateExtClient - creates an extclient +// CreateExtClient - creates and saves an extclient func CreateExtClient(extclient *models.ExtClient) error { - // lock because we need unique IPs and having it concurrent makes parallel calls result in same "unique" IPs + // lock because we may need unique IPs and having it concurrent makes parallel calls result in same "unique" IPs addressLock.Lock() defer addressLock.Unlock() @@ -219,12 +219,8 @@ func SaveExtClient(extclient *models.ExtClient) error { } // UpdateExtClient - updates an ext client with new values -func UpdateExtClient(old *models.ExtClient, update *models.CustomExtClient) (*models.ExtClient, error) { - new := old - err := DeleteExtClient(old.Network, old.ClientID) - if err != nil { - return new, err - } +func UpdateExtClient(old *models.ExtClient, update *models.CustomExtClient) models.ExtClient { + new := *old new.ClientID = update.ClientID if update.PublicKey != "" && old.PublicKey != update.PublicKey { new.PublicKey = update.PublicKey @@ -241,7 +237,7 @@ func UpdateExtClient(old *models.ExtClient, update *models.CustomExtClient) (*mo if update.DeniedACLs != nil && !reflect.DeepEqual(old.DeniedACLs, update.DeniedACLs) { new.DeniedACLs = update.DeniedACLs } - return new, CreateExtClient(new) + return new } // GetExtClientsByID - gets the clients of attached gateway From 84cc116813e4110b5c7104f35ae312eb01c545a7 Mon Sep 17 00:00:00 2001 From: Aceix Date: Wed, 2 Aug 2023 11:56:44 +0000 Subject: [PATCH 10/33] prune(NET-483): remove defunct host.internetgateway field (#2487) --- logic/hosts.go | 1 - models/api_host.go | 6 ------ models/host.go | 1 - models/node.go | 5 ----- 4 files changed, 13 deletions(-) diff --git a/logic/hosts.go b/logic/hosts.go index 904bdb5fc..e53ed730b 100644 --- a/logic/hosts.go +++ b/logic/hosts.go @@ -198,7 +198,6 @@ func UpdateHost(newHost, currentHost *models.Host) { newHost.Debug = currentHost.Debug newHost.Nodes = currentHost.Nodes newHost.PublicKey = currentHost.PublicKey - newHost.InternetGateway = currentHost.InternetGateway newHost.TrafficKeyPublic = currentHost.TrafficKeyPublic // changeable fields diff --git a/models/api_host.go b/models/api_host.go index c71147fc2..4558a984b 100644 --- a/models/api_host.go +++ b/models/api_host.go @@ -23,7 +23,6 @@ type ApiHost struct { EndpointIP string `json:"endpointip" yaml:"endpointip"` PublicKey string `json:"publickey"` MacAddress string `json:"macaddress"` - InternetGateway string `json:"internetgateway"` Nodes []string `json:"nodes"` IsDefault bool `json:"isdefault" yaml:"isdefault"` IsRelayed bool `json:"isrelayed" bson:"isrelayed" yaml:"isrelayed"` @@ -45,10 +44,6 @@ func (h *Host) ConvertNMHostToAPI() *ApiHost { a.Interfaces[i].AddressString = a.Interfaces[i].Address.String() } a.DefaultInterface = h.DefaultInterface - a.InternetGateway = h.InternetGateway.String() - if isEmptyAddr(a.InternetGateway) { - a.InternetGateway = "" - } a.IsStatic = h.IsStatic a.ListenPort = h.ListenPort a.MTU = h.MTU @@ -83,7 +78,6 @@ func (a *ApiHost) ConvertAPIHostToNMHost(currentHost *Host) *Host { h.Interface = currentHost.Interface h.Interfaces = currentHost.Interfaces h.DefaultInterface = currentHost.DefaultInterface - h.InternetGateway = currentHost.InternetGateway h.IsDocker = currentHost.IsDocker h.IsK8S = currentHost.IsK8S h.IsStatic = a.IsStatic diff --git a/models/host.go b/models/host.go index 70947ba29..acd866848 100644 --- a/models/host.go +++ b/models/host.go @@ -55,7 +55,6 @@ type Host struct { PublicKey wgtypes.Key `json:"publickey" yaml:"publickey"` MacAddress net.HardwareAddr `json:"macaddress" yaml:"macaddress"` TrafficKeyPublic []byte `json:"traffickeypublic" yaml:"traffickeypublic"` - InternetGateway net.UDPAddr `json:"internetgateway" yaml:"internetgateway"` Nodes []string `json:"nodes" yaml:"nodes"` Interfaces []Iface `json:"interfaces" yaml:"interfaces"` DefaultInterface string `json:"defaultinterface" yaml:"defaultinterface"` diff --git a/models/node.go b/models/node.go index ba236cbea..a4104ac39 100644 --- a/models/node.go +++ b/models/node.go @@ -484,10 +484,6 @@ func (ln *LegacyNode) ConvertToNewNode() (*Host, *Node) { host.PublicKey, _ = wgtypes.ParseKey(ln.PublicKey) host.MacAddress, _ = net.ParseMAC(ln.MacAddress) host.TrafficKeyPublic = ln.TrafficKeys.Mine - gateway, err := net.ResolveUDPAddr("udp", ln.InternetGateway) - if err == nil { - host.InternetGateway = *gateway - } id, _ := uuid.Parse(ln.ID) host.Nodes = append(host.Nodes, id.String()) host.Interfaces = ln.Interfaces @@ -572,7 +568,6 @@ func (n *Node) Legacy(h *Host, s *ServerConfig, net *Network) *LegacyNode { l.TrafficKeys.Mine = h.TrafficKeyPublic l.TrafficKeys.Server = s.TrafficKey l.FirewallInUse = h.FirewallInUse - l.InternetGateway = h.InternetGateway.String() l.Connected = formatBool(n.Connected) //l.PendingDelete = formatBool(n.PendingDelete) l.DefaultACL = n.DefaultACL From adefafbb394a02b9c9fee41922dccb9c13d18950 Mon Sep 17 00:00:00 2001 From: Abhishek K <32607604+abhishek9686@users.noreply.github.com> Date: Wed, 2 Aug 2023 22:01:41 +0530 Subject: [PATCH 11/33] don't reference host on err (#2493) --- mq/publishers.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mq/publishers.go b/mq/publishers.go index 2badddb84..ac195aa34 100644 --- a/mq/publishers.go +++ b/mq/publishers.go @@ -176,7 +176,7 @@ func PublishDNSUpdate(network string, dns models.DNSUpdate) error { for _, node := range nodes { host, err := logic.GetHost(node.HostID.String()) if err != nil { - logger.Log(0, "error retrieving host for dns update", host.ID.String(), err.Error()) + logger.Log(0, "error retrieving host for dns update", node.HostID.String(), err.Error()) continue } data, err := json.Marshal(dns) @@ -370,7 +370,7 @@ func getNodeDNS(network string) []models.DNSUpdate { for _, node := range nodes { host, err := logic.GetHost(node.HostID.String()) if err != nil { - logger.Log(0, "error retrieving host for dns update", host.ID.String(), err.Error()) + logger.Log(0, "error retrieving host for dns update", node.HostID.String(), err.Error()) continue } dns.Action = models.DNSInsert From c6c767562087950a635cffe6060d28f76b0a41f1 Mon Sep 17 00:00:00 2001 From: Abhishek K <32607604+abhishek9686@users.noreply.github.com> Date: Wed, 2 Aug 2023 22:01:59 +0530 Subject: [PATCH 12/33] deprecrate netclient install scripts (#2490) --- scripts/netclient-install.ps1 | 78 --------- scripts/netclient-install.sh | 307 ---------------------------------- 2 files changed, 385 deletions(-) delete mode 100644 scripts/netclient-install.ps1 delete mode 100755 scripts/netclient-install.sh diff --git a/scripts/netclient-install.ps1 b/scripts/netclient-install.ps1 deleted file mode 100644 index 1c65a8ded..000000000 --- a/scripts/netclient-install.ps1 +++ /dev/null @@ -1,78 +0,0 @@ -new-module -name netclient-install -scriptblock { - $ErrorActionPreference = "Stop" - - function Quit { - param( - $Text - ) - Write-Host "Exiting: " $Text - Break Script - } - Function Netclient-Install() { - param ($version='latest', $token) - - if($token -eq $null -or $token -eq ""){ - Quit "-token required" - } - - $software = "WireGuard"; - $installed = (Get-ItemProperty HKLM:\Software\Microsoft\Windows\CurrentVersion\Uninstall\* | Where { $_.DisplayName -eq $software }) -ne $null - - If(-Not $installed) { - Write-Host "'$software' is NOT installed. installing..."; - $url = "https://download.wireguard.com/windows-client/wireguard-installer.exe" - $outpath = "$env:userprofile\Downloads\wireguard-installer.exe" - Invoke-WebRequest -Uri $url -OutFile $outpath - $args = @("Comma","Separated","Arguments") - Start-Process -Filepath "$env:userprofile\Downloads\wireguard-installer.exe" -ArgumentList $args -Wait - $software = "WireGuard"; - $installed = (Get-ItemProperty HKLM:\Software\Microsoft\Windows\CurrentVersion\Uninstall\* | Where { $_.DisplayName -eq $software }) -ne $null - If(-Not $installed) { - Quit "Could not install WireGuard" - } else { - # $env:Path += (";" + $env:ProgramFiles + "\WireGuard") - Write-Host "'$software' is installed." - } - } else { - Write-Host "'$software' is installed." - } - $outpath = ""; - if (Test-Path -Path "C:\ProgramData\Netclient\bin\netclient.exe") { - $outpath = "C:\ProgramData\Netclient\bin\netclient.exe"; - } else { - $outpath = "$env:userprofile\Downloads\netclient.exe" - Write-Host "'netclient.exe' is NOT installed. installing..."; - Write-Host "https://github.com/gravitl/netmaker/releases/download/$version/netclient.exe"; - $url = "https://github.com/gravitl/netmaker/releases/download/$version/netclient.exe" - Invoke-WebRequest -Uri $url -OutFile $outpath - $loc = Get-Location - Copy-Item -Path "$env:userprofile\Downloads\netclient.exe" -Destination "$loc\netclient.exe" - } - $runNum = "one" - foreach ($run in $runNum) { - - $NetArgs = @("join","-t",$token) - Start-Process -Filepath $outpath -ArgumentList $NetArgs -Wait - Add-MpPreference -ExclusionPath "C:\ProgramData\Netclient" - - if ((Get-Command "netclient.exe" -ErrorAction SilentlyContinue) -eq $null) { - if (-not (Test-Path -Path "C:\ProgramData\Netclient\bin\netclient.exe")) { - New-Item -Path "C:\ProgramData\Netclient" -Name "bin" -ItemType "directory" - Move-Item -Path "$env:userprofile\Downloads\netclient.exe" -Destination "C:\ProgramData\Netclient\bin\netclient.exe" - $oldpath = (Get-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH).path - $newpath = "$oldpath;C:\ProgramData\Netclient\bin" - Set-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH -Value $newPath - $env:Path += ";C:\ProgramData\Netclient\bin" - } - } - #if($run -eq "one"){ - # Write-Host "re-running setup to confirm all components are installed." - # Start-Sleep -s 1 - #} - - } - Start-Sleep -s 5 - Write-Host "'netclient' is installed." - } -} - diff --git a/scripts/netclient-install.sh b/scripts/netclient-install.sh deleted file mode 100755 index 3df02c7bb..000000000 --- a/scripts/netclient-install.sh +++ /dev/null @@ -1,307 +0,0 @@ -#!/bin/sh - -if [ $(id -u) -ne 0 ]; then - echo "This script must be run as root" - exit 1 -fi - -echo "checking dependencies..." - -OS=$(uname) - -if [ -f /etc/debian_version ]; then - dependencies="wireguard wireguard-tools" - update_cmd='apt update' - install_cmd='apt-get install -y' -elif [ -f /etc/alpine-release ]; then - dependencies="wireguard" - update_cmd='apk update' - install_cmd='apk --update add' -elif [ -f /etc/centos-release ]; then - dependencies="wireguard" - update_cmd='yum update' - install_cmd='yum install -y' -elif [ -f /etc/fedora-release ]; then - dependencies="wireguard" - update_cmd='dnf update' - install_cmd='dnf install -y' -elif [ -f /etc/redhat-release ]; then - dependencies="wireguard" - update_cmd='yum update' - install_cmd='yum install -y' -elif [ -f /etc/arch-release ]; then - dependecies="wireguard-tools" - update_cmd='pacman -Sy' - install_cmd='pacman -S --noconfirm' -elif [ "${OS}" = "FreeBSD" ]; then - dependencies="wireguard wget" - update_cmd='pkg update' - install_cmd='pkg install -y' -elif [ -f /etc/turris-version ]; then - dependencies="wireguard-tools bash" - OS="TurrisOS" - update_cmd='opkg update' - install_cmd='opkg install' -elif [ -f /etc/openwrt_release ]; then - dependencies="wireguard-tools bash" - OS="OpenWRT" - update_cmd='opkg update' - install_cmd='opkg install' -else - install_cmd='' -fi - -if [ -z "${install_cmd}" ]; then - echo "OS unsupported for automatic dependency install" - exit 1 -fi - -${update_cmd} - -set -- $dependencies -while [ -n "$1" ]; do - echo $1 - if [ "${OS}" = "FreeBSD" ]; then - is_installed=$(pkg check -d $1 | grep "Checking" | grep "done") - if [ "$is_installed" != "" ]; then - echo " " $1 is installed - else - echo " " $1 is not installed. Attempting install. - ${install_cmd} $1 - sleep 5 - is_installed=$(pkg check -d $1 | grep "Checking" | grep "done") - if [ "$is_installed" != "" ]; then - echo " " $1 is installed - elif [ -x "$(command -v $1)" ]; then - echo " " $1 is installed - else - echo " " FAILED TO INSTALL $1 - echo " " This may break functionality. - fi - fi - else - if [ "${OS}" = "OpenWRT" ] || [ "${OS}" = "TurrisOS" ]; then - is_installed=$(opkg list-installed $1 | grep $1) - else - is_installed=$(dpkg-query -W --showformat='${Status}\n' $1 | grep "install ok installed") - fi - if [ "${is_installed}" != "" ]; then - echo " " $1 is installed - else - echo " " $1 is not installed. Attempting install. - ${install_cmd} $1 - sleep 5 - if [ "${OS}" = "OpenWRT" ] || [ "${OS}" = "TurrisOS" ]; then - is_installed=$(opkg list-installed $1 | grep $1) - else - is_installed=$(dpkg-query -W --showformat='${Status}\n' $1 | grep "install ok installed") - fi - if [ "${is_installed}" != "" ]; then - echo " " $1 is installed - elif [ -x "$(command -v $1)" ]; then - echo " " $1 is installed - else - echo " " FAILED TO INSTALL $1 - echo " " This may break functionality. - fi - fi - fi - shift -done - -set -e - -[ -z "$KEY" ] && KEY=nokey; -[ -z "$VERSION" ] && echo "no \$VERSION provided, fallback to latest" && VERSION=latest; -[ "latest" != "$VERSION" ] && [ "v" != `echo $VERSION | cut -c1` ] && VERSION="v$VERSION" -[ -z "$NAME" ] && NAME=""; - -dist=netclient - -echo "OS Version = $(uname)" -echo "Netclient Version = $VERSION" - -case $(uname | tr A-Z a-z) in - linux*) - if [ -z "$CPU_ARCH" ]; then - CPU_ARCH=$(uname -m) - fi - case $CPU_ARCH in - amd64) - dist=netclient - ;; - x86_64) - dist=netclient - ;; - arm64) - dist=netclient-arm64 - ;; - aarch64) - dist=netclient-arm64 - ;; - armv6l) - dist=netclient-arm6 - ;; - armv7l) - dist=netclient-arm7 - ;; - arm*) - dist=netclient-$CPU_ARCH - ;; - mipsle) - dist=netclient-mipsle - ;; - mips) - #If binary in the below condition is not compatible with your hardware, retry with other netclient-mips* binaries. - if [[ `printf '\0\1' | hexdump -e '/2 "%04x"'` -eq 0100 ]]; then - #Little Endian, tested and confirmed in GL-MT1300 OS "OpenWrt 19.07.8" - dist=netclient-mipsle-softfloat - else - #Big Endian, tested and confirmed in DSL-2750U OS "OpenWrt 22.03.2" - dist=netclient-mips-softfloat - fi - ;; - *) - fatal "$CPU_ARCH : cpu architecture not supported" - esac - ;; - darwin) - dist=netclient-darwin - ;; - Darwin) - dist=netclient-darwin - ;; - freebsd*) - if [ -z "$CPU_ARCH" ]; then - CPU_ARCH=$(uname -m) - fi - case $CPU_ARCH in - amd64) - dist=netclient-freebsd - ;; - x86_64) - dist=netclient-freebsd - ;; - arm64) - dist=netclient-freebsd-arm64 - ;; - aarch64) - dist=netclient-freebsd-arm64 - ;; - armv7l) - dist=netclient-freebsd-arm7 - ;; - arm*) - dist=netclient-freebsd-$CPU_ARCH - ;; - *) - fatal "$CPU_ARCH : cpu architecture not supported" - esac - ;; -esac - -echo "Binary = $dist" - -url="https://github.com/gravitl/netmaker/releases/download/$VERSION/$dist" -curl_opts='-nv' -if [ "${OS}" = "OpenWRT" ] || [ "${OS}" = "TurrisOS" ]; then - curl_opts='-q' -fi - -if curl --output /dev/null --silent --head --fail "$url"; then - echo "Downloading $dist $VERSION" - wget $curl_opts -O netclient $url -else - echo "Downloading $dist latest" - wget $curl_opts -O netclient https://github.com/gravitl/netmaker/releases/latest/download/$dist -fi - -chmod +x netclient - -EXTRA_ARGS="" -if [ "${OS}" = "OpenWRT" ] || [ "${OS}" = "TurrisOS" ]; then - EXTRA_ARGS="--daemon=off" -fi - -if [ "${KEY}" != "nokey" ]; then - if [ -z "${NAME}" ]; then - ./netclient join -t $KEY $EXTRA_ARGS - else - ./netclient join -t $KEY --name $NAME $EXTRA_ARGS - fi -fi - -if [ "${OS}" = "FreeBSD" ]; then - if ! [ -x /usr/sbin/netclient ]; then - echo "Moving netclient executable to \"/usr/sbin/netclient\"" - mv netclient /usr/sbin - else - echo "Netclient already present." - fi -fi - -if [ "${OS}" = "OpenWRT" ] || [ "${OS}" = "TurrisOS" ]; then - mv ./netclient /sbin/netclient - - if [ "${OS}" = "TurrisOS" ]; then - url="https://raw.githubusercontent.com/gravitl/netmaker/$VERSION/scripts/openwrt-daemon.sh" - if curl --output /dev/null --silent --head --fail $url; then - wget $curl_opts -O netclient.service.tmp $url - else - wget $curl_opts -O netclient.service.tmp https://raw.githubusercontent.com/gravitl/netmaker/master/scripts/openwrt-daemon.sh - fi - elif [ "${OS}" = "OpenWRT" ] && [ "$CPU_ARCH" = "mips" ]; then - wget $curl_opts -O netclient.service.tmp https://raw.githubusercontent.com/gravitl/netmaker/master/scripts/openwrt-daemon.sh - else - cat << 'END_OF_FILE' > ./netclient.service.tmp -#!/bin/sh /etc/rc.common - -EXTRA_COMMANDS="status" -EXTRA_HELP=" status Check service is running" -START=99 - -LOG_FILE="/tmp/netclient.logs" - -start() { - if [ ! -f "${LOG_FILE}" ];then - touch "${LOG_FILE}" - fi - local PID=$(ps|grep "netclient daemon"|grep -v grep|awk '{print $1}') - if [ "${PID}" ];then - echo "service is running" - return - fi - bash -c "do /sbin/netclient daemon >> ${LOG_FILE} 2>&1;\ - if [ $(ls -l ${LOG_FILE}|awk '{print $5}') -gt 10240000 ];then tar zcf "${LOG_FILE}.tar" -C / "tmp/netclient.logs" && > $LOG_FILE;fi;done &" - echo "start" -} - -stop() { - pids=$(ps|grep "netclient daemon"|grep -v grep|awk '{print $1}') - for i in "${pids[@]}" - do - if [ "${i}" ];then - kill "${i}" - fi - done - echo "stop" -} - -status() { - local PID=$(ps|grep "netclient daemon"|grep -v grep|awk '{print $1}') - if [ "${PID}" ];then - echo -e "netclient[${PID}] is running \n" - else - echo -e "netclient is not running \n" - fi -} - -END_OF_FILE - fi - mv ./netclient.service.tmp /etc/init.d/netclient - chmod +x /etc/init.d/netclient - /etc/init.d/netclient enable - /etc/init.d/netclient start -else - rm -f netclient -fi \ No newline at end of file From d3f6506e26631ae9dc3fd56d1ecba94945d20610 Mon Sep 17 00:00:00 2001 From: Matthew R Kasun Date: Wed, 2 Aug 2023 12:32:25 -0400 Subject: [PATCH 13/33] Net 500: validate network parameter passed to node endpoints (#2480) * enforce unique names for ext client names * only check for unique id on creation * check for unique id if changed * validate network parameter passed to node endpoints --------- Co-authored-by: Abhishek K <32607604+abhishek9686@users.noreply.github.com> --- controllers/node.go | 72 +++++++++++++++++++++++++++++---------------- 1 file changed, 47 insertions(+), 25 deletions(-) diff --git a/controllers/node.go b/controllers/node.go index 3b4aceed6..64055ebd4 100644 --- a/controllers/node.go +++ b/controllers/node.go @@ -16,6 +16,7 @@ import ( "github.com/gravitl/netmaker/mq" "github.com/gravitl/netmaker/servercfg" "golang.org/x/crypto/bcrypt" + "golang.org/x/exp/slog" ) var hostIDHeader = "host-id" @@ -373,11 +374,10 @@ func getNode(w http.ResponseWriter, r *http.Request) { var params = mux.Vars(r) nodeid := params["nodeid"] - node, err := logic.GetNodeByID(nodeid) + + node, err := validateParams(nodeid, params["network"]) if err != nil { - logger.Log(0, r.Header.Get("user"), - fmt.Sprintf("error fetching node [ %s ] info: %v", nodeid, err)) - logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal")) + logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest")) return } host, err := logic.GetHost(node.HostID.String()) @@ -442,16 +442,20 @@ func getNode(w http.ResponseWriter, r *http.Request) { func createEgressGateway(w http.ResponseWriter, r *http.Request) { var gateway models.EgressGatewayRequest var params = mux.Vars(r) - w.Header().Set("Content-Type", "application/json") - err := json.NewDecoder(r.Body).Decode(&gateway) + node, err := validateParams(params["nodeid"], params["network"]) if err != nil { + logic.ReturnErrorResponse(w, r, logic.FormatError(err, "bad request")) + return + } + w.Header().Set("Content-Type", "application/json") + if err := json.NewDecoder(r.Body).Decode(&gateway); err != nil { logger.Log(0, r.Header.Get("user"), "error decoding request body: ", err.Error()) logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest")) return } gateway.NetID = params["network"] gateway.NodeID = params["nodeid"] - node, err := logic.CreateEgressGateway(gateway) + node, err = logic.CreateEgressGateway(gateway) if err != nil { logger.Log(0, r.Header.Get("user"), fmt.Sprintf("failed to create egress gateway on node [%s] on network [%s]: %v", @@ -487,7 +491,12 @@ func deleteEgressGateway(w http.ResponseWriter, r *http.Request) { var params = mux.Vars(r) nodeid := params["nodeid"] netid := params["network"] - node, err := logic.DeleteEgressGateway(netid, nodeid) + node, err := validateParams(nodeid, netid) + if err != nil { + logic.ReturnErrorResponse(w, r, logic.FormatError(err, "bad request")) + return + } + node, err = logic.DeleteEgressGateway(netid, nodeid) if err != nil { logger.Log(0, r.Header.Get("user"), fmt.Sprintf("failed to delete egress gateway on node [%s] on network [%s]: %v", @@ -524,10 +533,14 @@ func createIngressGateway(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") nodeid := params["nodeid"] netid := params["network"] + node, err := validateParams(nodeid, netid) + if err != nil { + logic.ReturnErrorResponse(w, r, logic.FormatError(err, "bad request")) + return + } var request models.IngressRequest json.NewDecoder(r.Body).Decode(&request) - - node, err := logic.CreateIngressGateway(netid, nodeid, request) + node, err = logic.CreateIngressGateway(netid, nodeid, request) if err != nil { logger.Log(0, r.Header.Get("user"), fmt.Sprintf("failed to create ingress gateway on node [%s] on network [%s]: %v", @@ -566,6 +579,11 @@ func deleteIngressGateway(w http.ResponseWriter, r *http.Request) { var params = mux.Vars(r) nodeid := params["nodeid"] netid := params["network"] + node, err := validateParams(nodeid, netid) + if err != nil { + logic.ReturnErrorResponse(w, r, logic.FormatError(err, "bad request")) + return + } node, wasFailover, removedClients, err := logic.DeleteIngressGateway(nodeid) if err != nil { logger.Log(0, r.Header.Get("user"), @@ -623,14 +641,11 @@ func updateNode(w http.ResponseWriter, r *http.Request) { //start here nodeid := params["nodeid"] - currentNode, err := logic.GetNodeByID(nodeid) + currentNode, err := validateParams(nodeid, params["network"]) if err != nil { - logger.Log(0, r.Header.Get("user"), - fmt.Sprintf("error fetching node [ %s ] info: %v", nodeid, err)) - logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal")) + logic.ReturnErrorResponse(w, r, logic.FormatError(err, "bad request")) return } - var newData models.ApiNode // we decode our body request params err = json.NewDecoder(r.Body).Decode(&newData) @@ -721,19 +736,13 @@ func deleteNode(w http.ResponseWriter, r *http.Request) { // get params var params = mux.Vars(r) var nodeid = params["nodeid"] - forceDelete := r.URL.Query().Get("force") == "true" - fromNode := r.Header.Get("requestfrom") == "node" - node, err := logic.GetNodeByID(nodeid) + node, err := validateParams(nodeid, params["network"]) if err != nil { - if logic.CheckAndRemoveLegacyNode(nodeid) { - logger.Log(0, "removed legacy node", nodeid) - logic.ReturnSuccessResponse(w, r, nodeid+" deleted.") - } else { - logger.Log(0, "error retrieving node to delete", err.Error()) - logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest")) - } + logic.ReturnErrorResponse(w, r, logic.FormatError(err, "bad request")) return } + forceDelete := r.URL.Query().Get("force") == "true" + fromNode := r.Header.Get("requestfrom") == "node" if r.Header.Get("ismaster") != "yes" { username := r.Header.Get("user") if username != "" && !doesUserOwnNode(username, params["network"], nodeid) { @@ -816,3 +825,16 @@ func doesUserOwnNode(username, network, nodeID string) bool { return logic.StringSliceContains(netUser.Nodes, nodeID) } + +func validateParams(nodeid, netid string) (models.Node, error) { + node, err := logic.GetNodeByID(nodeid) + if err != nil { + slog.Error("error fetching node", "node", nodeid, "error", err.Error()) + return node, fmt.Errorf("error fetching node during parameter validation: %v", err) + } + if node.Network != netid { + slog.Error("network url param does not match node id", "url nodeid", netid, "node", node.Network) + return node, fmt.Errorf("network url param does not match node network") + } + return node, nil +} From 29f7b483472a91cef930d0cd1d1b5d62c90d7225 Mon Sep 17 00:00:00 2001 From: Farukh Khan Date: Thu, 3 Aug 2023 14:43:40 +0800 Subject: [PATCH 14/33] NET-513 (#2492) nm-certs.sh now requests certificate for EE and CE edition domains accordingly. --- scripts/nm-certs.sh | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/scripts/nm-certs.sh b/scripts/nm-certs.sh index cd27f0ab9..1d99710bf 100755 --- a/scripts/nm-certs.sh +++ b/scripts/nm-certs.sh @@ -27,20 +27,34 @@ if [ -n "$(docker ps | grep caddy)" ]; then docker-compose -f /root/docker-compose.yml stop caddy fi -CERTBOT_PARAMS=$(cat <"$SCRIPT_DIR/certbot-entry.sh" From 2bfeb4cfe2dd2c41e6a3c00841662421d7e8b51a Mon Sep 17 00:00:00 2001 From: Gabriel de Souza Seibel Date: Thu, 3 Aug 2023 03:46:58 -0300 Subject: [PATCH 15/33] [NET-404] Run in limited mode when ee checks fail (#2474) * Add limited http handlers functionality to rest handler * Export ee.errValidation (ee.ErrValidation) * Export a fatal error handled by the hook manager * Export a new status variable for unlicensed server * Mark server as unlicensed when ee checks fail * Handle license validation failures with a (re)boot in a limited state * Revert "Export a fatal error handled by the hook manager" This reverts commit 069c21974a8d36e889c73ad78023448d787d62a5. * Revert "Export ee.errValidation (ee.ErrValidation)" This reverts commit 59dbab8c79773ca5d879f28cbaf53f3dd4297b9b. * Revert "Add limited http handlers functionality to rest handler" This reverts commit e2f1f28facaca54713db76a588839cd2733cf673. * Revert "Handle license validation failures with a (re)boot in a limited state" This reverts commit 58cfbbaf522a1345aac1fa67964ebff0a6d60cd8. * Revert "Mark server as unlicensed when ee checks fail" This reverts commit 77c6dbdd3c9cfa6e7d6becedef6251e8617ae367. * Handle license validation failures with a middleware * Forbid responses if unlicensed ee and not in status api * Remove unused func --- controllers/controller.go | 7 +++++ controllers/server.go | 23 +++++++------- ee/ee_controllers/middleware.go | 17 +++++++++++ ee/initialize.go | 15 ++++++--- ee/license.go | 54 ++++++++++++++++++++------------- logic/timer.go | 12 +++++--- servercfg/serverconf.go | 5 +-- 7 files changed, 90 insertions(+), 43 deletions(-) create mode 100644 ee/ee_controllers/middleware.go diff --git a/controllers/controller.go b/controllers/controller.go index 7abef20b3..8e54d38b1 100644 --- a/controllers/controller.go +++ b/controllers/controller.go @@ -14,6 +14,9 @@ import ( "github.com/gravitl/netmaker/servercfg" ) +// HttpMiddlewares - middleware functions for REST interactions +var HttpMiddlewares []mux.MiddlewareFunc + // HttpHandlers - handler functions for REST interactions var HttpHandlers = []interface{}{ nodeHandlers, @@ -42,6 +45,10 @@ func HandleRESTRequests(wg *sync.WaitGroup, ctx context.Context) { originsOk := handlers.AllowedOrigins(strings.Split(servercfg.GetAllowedOrigin(), ",")) methodsOk := handlers.AllowedMethods([]string{http.MethodGet, http.MethodPut, http.MethodPost, http.MethodDelete}) + for _, middleware := range HttpMiddlewares { + r.Use(middleware) + } + for _, handler := range HttpHandlers { handler.(func(*mux.Router))(r) } diff --git a/controllers/server.go b/controllers/server.go index e9aa1b760..79c8e21f0 100644 --- a/controllers/server.go +++ b/controllers/server.go @@ -68,22 +68,21 @@ func getUsage(w http.ResponseWriter, r *http.Request) { // Responses: // 200: serverConfigResponse func getStatus(w http.ResponseWriter, r *http.Request) { - // TODO - // - check health of broker type status struct { - DB bool `json:"db_connected"` - Broker bool `json:"broker_connected"` - Usage struct { - Hosts int `json:"hosts"` - Clients int `json:"clients"` - Networks int `json:"networks"` - Users int `json:"users"` - } `json:"usage"` + DB bool `json:"db_connected"` + Broker bool `json:"broker_connected"` + LicenseError string `json:"license_error"` + } + + licenseErr := "" + if servercfg.ErrLicenseValidation != nil { + licenseErr = servercfg.ErrLicenseValidation.Error() } currentServerStatus := status{ - DB: database.IsConnected(), - Broker: mq.IsConnected(), + DB: database.IsConnected(), + Broker: mq.IsConnected(), + LicenseError: licenseErr, } w.Header().Set("Content-Type", "application/json") diff --git a/ee/ee_controllers/middleware.go b/ee/ee_controllers/middleware.go new file mode 100644 index 000000000..dc853524b --- /dev/null +++ b/ee/ee_controllers/middleware.go @@ -0,0 +1,17 @@ +package ee_controllers + +import ( + "github.com/gravitl/netmaker/logic" + "github.com/gravitl/netmaker/servercfg" + "net/http" +) + +func OnlyServerAPIWhenUnlicensedMiddleware(handler http.Handler) http.Handler { + return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if servercfg.ErrLicenseValidation != nil && request.URL.Path != "/api/server/status" { + logic.ReturnErrorResponse(writer, request, logic.FormatError(servercfg.ErrLicenseValidation, "forbidden")) + return + } + handler.ServeHTTP(writer, request) + }) +} diff --git a/ee/initialize.go b/ee/initialize.go index 455ee59df..e779147bf 100644 --- a/ee/initialize.go +++ b/ee/initialize.go @@ -7,10 +7,10 @@ import ( controller "github.com/gravitl/netmaker/controllers" "github.com/gravitl/netmaker/ee/ee_controllers" eelogic "github.com/gravitl/netmaker/ee/logic" - "github.com/gravitl/netmaker/logger" "github.com/gravitl/netmaker/logic" "github.com/gravitl/netmaker/models" "github.com/gravitl/netmaker/servercfg" + "golang.org/x/exp/slog" ) // InitEE - Initialize EE Logic @@ -18,6 +18,10 @@ func InitEE() { setIsEnterprise() servercfg.Is_EE = true models.SetLogo(retrieveEELogo()) + controller.HttpMiddlewares = append( + controller.HttpMiddlewares, + ee_controllers.OnlyServerAPIWhenUnlicensedMiddleware, + ) controller.HttpHandlers = append( controller.HttpHandlers, ee_controllers.MetricHandlers, @@ -27,8 +31,11 @@ func InitEE() { ) logic.EnterpriseCheckFuncs = append(logic.EnterpriseCheckFuncs, func() { // == License Handling == - ValidateLicense() - logger.Log(0, "proceeding with Paid Tier license") + if err := ValidateLicense(); err != nil { + slog.Error(err.Error()) + return + } + slog.Info("proceeding with Paid Tier license") logic.SetFreeTierForTelemetry(false) // == End License Handling == AddLicenseHooks() @@ -48,7 +55,7 @@ func resetFailover() { for _, net := range nets { err = eelogic.ResetFailover(net.NetID) if err != nil { - logger.Log(0, "failed to reset failover on network", net.NetID, ":", err.Error()) + slog.Error("failed to reset failover", "network", net.NetID, "error", err.Error()) } } } diff --git a/ee/license.go b/ee/license.go index dfbb36eca..5f05df432 100644 --- a/ee/license.go +++ b/ee/license.go @@ -12,7 +12,6 @@ import ( "golang.org/x/exp/slog" "io" "net/http" - "os" "time" "github.com/gravitl/netmaker/database" @@ -44,29 +43,40 @@ func AddLicenseHooks() { } } -// ValidateLicense - the initial license check for netmaker server +// ValidateLicense - the initial and periodic license check for netmaker server // checks if a license is valid + limits are not exceeded -// if license is free_tier and limits exceeds, then server should terminate -// if license is not valid, server should terminate -func ValidateLicense() error { +// if license is free_tier and limits exceeds, then function should error +// if license is not valid, function should error +func ValidateLicense() (err error) { + defer func() { + if err != nil { + err = fmt.Errorf("%w: %s", errValidation, err.Error()) + servercfg.ErrLicenseValidation = err + } + }() + licenseKeyValue := servercfg.GetLicenseKey() netmakerTenantID := servercfg.GetNetmakerTenantID() slog.Info("proceeding with Netmaker license validation...") if len(licenseKeyValue) == 0 { - failValidation(errors.New("empty license-key (LICENSE_KEY environment variable)")) + err = errors.New("empty license-key (LICENSE_KEY environment variable)") + return err } if len(netmakerTenantID) == 0 { - failValidation(errors.New("empty tenant-id (NETMAKER_TENANT_ID environment variable)")) + err = errors.New("empty tenant-id (NETMAKER_TENANT_ID environment variable)") + return err } apiPublicKey, err := getLicensePublicKey(licenseKeyValue) if err != nil { - failValidation(fmt.Errorf("failed to get license public key: %w", err)) + err = fmt.Errorf("failed to get license public key: %w", err) + return err } tempPubKey, tempPrivKey, err := FetchApiServerKeys() if err != nil { - failValidation(fmt.Errorf("failed to fetch api server keys: %w", err)) + err = fmt.Errorf("failed to fetch api server keys: %w", err) + return err } licenseSecret := LicenseSecret{ @@ -76,35 +86,42 @@ func ValidateLicense() error { secretData, err := json.Marshal(&licenseSecret) if err != nil { - failValidation(fmt.Errorf("failed to marshal license secret: %w", err)) + err = fmt.Errorf("failed to marshal license secret: %w", err) + return err } encryptedData, err := ncutils.BoxEncrypt(secretData, apiPublicKey, tempPrivKey) if err != nil { - failValidation(fmt.Errorf("failed to encrypt license secret data: %w", err)) + err = fmt.Errorf("failed to encrypt license secret data: %w", err) + return err } validationResponse, err := validateLicenseKey(encryptedData, tempPubKey) if err != nil { - failValidation(fmt.Errorf("failed to validate license key: %w", err)) + err = fmt.Errorf("failed to validate license key: %w", err) + return err } if len(validationResponse) == 0 { - failValidation(errors.New("empty validation response")) + err = errors.New("empty validation response") + return err } var licenseResponse ValidatedLicense if err = json.Unmarshal(validationResponse, &licenseResponse); err != nil { - failValidation(fmt.Errorf("failed to unmarshal validation response: %w", err)) + err = fmt.Errorf("failed to unmarshal validation response: %w", err) + return err } respData, err := ncutils.BoxDecrypt(base64decode(licenseResponse.EncryptedLicense), apiPublicKey, tempPrivKey) if err != nil { - failValidation(fmt.Errorf("failed to decrypt license: %w", err)) + err = fmt.Errorf("failed to decrypt license: %w", err) + return err } license := LicenseKey{} if err = json.Unmarshal(respData, &license); err != nil { - failValidation(fmt.Errorf("failed to unmarshal license key: %w", err)) + err = fmt.Errorf("failed to unmarshal license key: %w", err) + return err } slog.Info("License validation succeeded!") @@ -158,11 +175,6 @@ func FetchApiServerKeys() (pub *[32]byte, priv *[32]byte, err error) { return pub, priv, nil } -func failValidation(err error) { - slog.Error(errValidation.Error(), "error", err) - os.Exit(0) -} - func getLicensePublicKey(licensePubKeyEncoded string) (*[32]byte, error) { decodedPubKey := base64decode(licensePubKeyEncoded) return ncutils.ConvertBytesToKey(decodedPubKey) diff --git a/logic/timer.go b/logic/timer.go index 89070f2fe..7a21a3785 100644 --- a/logic/timer.go +++ b/logic/timer.go @@ -3,10 +3,11 @@ package logic import ( "context" "fmt" + "github.com/gravitl/netmaker/logger" + "golang.org/x/exp/slog" "sync" "time" - "github.com/gravitl/netmaker/logger" "github.com/gravitl/netmaker/models" ) @@ -52,7 +53,7 @@ func StartHookManager(ctx context.Context, wg *sync.WaitGroup) { for { select { case <-ctx.Done(): - logger.Log(0, "## Stopping Hook Manager") + slog.Error("## Stopping Hook Manager") return case newhook := <-HookManagerCh: wg.Add(1) @@ -70,7 +71,9 @@ func addHookWithInterval(ctx context.Context, wg *sync.WaitGroup, hook func() er case <-ctx.Done(): return case <-ticker.C: - hook() + if err := hook(); err != nil { + slog.Error(err.Error()) + } } } @@ -85,6 +88,7 @@ var timeHooks = []interface{}{ } func loggerDump() error { + // TODO use slog? logger.DumpFile(fmt.Sprintf("data/netmaker.log.%s", time.Now().Format(logger.TimeFormatDay))) return nil } @@ -93,7 +97,7 @@ func loggerDump() error { func runHooks() { for _, hook := range timeHooks { if err := hook.(func() error)(); err != nil { - logger.Log(1, "error occurred when running timer function:", err.Error()) + slog.Error("error occurred when running timer function", "error", err.Error()) } } } diff --git a/servercfg/serverconf.go b/servercfg/serverconf.go index 18170a7eb..f4e70030e 100644 --- a/servercfg/serverconf.go +++ b/servercfg/serverconf.go @@ -18,8 +18,9 @@ import ( const EmqxBrokerType = "emqx" var ( - Version = "dev" - Is_EE = false + Version = "dev" + Is_EE = false + ErrLicenseValidation error ) // SetHost - sets the host ip From ff69d12cebe44cbcdd866794701c1a1f300e7d20 Mon Sep 17 00:00:00 2001 From: Aceix Date: Fri, 4 Aug 2023 07:52:47 +0000 Subject: [PATCH 16/33] feat(NET-449): add sync feature to request a host pull from server (#2491) --- controllers/hosts.go | 42 ++++++++++++++++++++++++++++++++++++++++++ models/host.go | 2 ++ 2 files changed, 44 insertions(+) diff --git a/controllers/hosts.go b/controllers/hosts.go index 764fe5c21..f712c25d1 100644 --- a/controllers/hosts.go +++ b/controllers/hosts.go @@ -13,12 +13,14 @@ import ( "github.com/gravitl/netmaker/mq" "github.com/gravitl/netmaker/servercfg" "golang.org/x/crypto/bcrypt" + "golang.org/x/exp/slog" ) func hostHandlers(r *mux.Router) { r.HandleFunc("/api/hosts", logic.SecurityCheck(false, http.HandlerFunc(getHosts))).Methods(http.MethodGet) r.HandleFunc("/api/hosts/keys", logic.SecurityCheck(true, http.HandlerFunc(updateAllKeys))).Methods(http.MethodPut) r.HandleFunc("/api/hosts/{hostid}/keys", logic.SecurityCheck(true, http.HandlerFunc(updateKeys))).Methods(http.MethodPut) + r.HandleFunc("/api/hosts/{hostid}/sync", logic.SecurityCheck(true, http.HandlerFunc(syncHost))).Methods(http.MethodPost) r.HandleFunc("/api/hosts/{hostid}", logic.SecurityCheck(true, http.HandlerFunc(updateHost))).Methods(http.MethodPut) r.HandleFunc("/api/hosts/{hostid}", logic.SecurityCheck(true, http.HandlerFunc(deleteHost))).Methods(http.MethodDelete) r.HandleFunc("/api/hosts/{hostid}/networks/{network}", logic.SecurityCheck(true, http.HandlerFunc(addHostToNetwork))).Methods(http.MethodPost) @@ -583,3 +585,43 @@ func updateKeys(w http.ResponseWriter, r *http.Request) { logger.Log(2, r.Header.Get("user"), "updated key on host", host.Name) w.WriteHeader(http.StatusOK) } + +// swagger:route POST /api/hosts/{hostId}/sync host syncHost +// +// Requests a host to pull. +// +// Schemes: https +// +// Security: +// oauth +// +// Responses: +// 200: networkBodyResponse +func syncHost(w http.ResponseWriter, r *http.Request) { + hostId := mux.Vars(r)["hostid"] + + var errorResponse = models.ErrorResponse{} + w.Header().Set("Content-Type", "application/json") + + host, err := logic.GetHost(hostId) + if err != nil { + slog.Error("failed to retrieve host", "user", r.Header.Get("user"), "error", err) + errorResponse.Code = http.StatusBadRequest + errorResponse.Message = err.Error() + logic.ReturnErrorResponse(w, r, errorResponse) + return + } + + go func() { + hostUpdate := models.HostUpdate{ + Action: models.RequestPull, + Host: *host, + } + if err = mq.HostUpdate(&hostUpdate); err != nil { + slog.Error("failed to send host pull request", "host", host.ID.String(), "error", err) + } + }() + + slog.Info("requested host pull", "user", r.Header.Get("user"), "host", host.ID) + w.WriteHeader(http.StatusOK) +} diff --git a/models/host.go b/models/host.go index acd866848..2ab90e16d 100644 --- a/models/host.go +++ b/models/host.go @@ -107,6 +107,8 @@ const ( RegisterWithTurn = "REGISTER_WITH_TURN" // UpdateKeys - update wireguard private/public keys UpdateKeys = "UPDATE_KEYS" + // RequestPull - request a pull from a host + RequestPull = "REQ_PULL" ) // SignalAction - turn peer signal action From 22f1464b6fdaf8ec5940922efe15627e5a3ec2e3 Mon Sep 17 00:00:00 2001 From: Aceix Date: Tue, 8 Aug 2023 10:29:02 +0000 Subject: [PATCH 17/33] fix(NET-486): change client name length validation (#2498) set limit to 5<=x<=32 --- controllers/regex.go | 2 +- controllers/regex_test.go | 20 +++++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/controllers/regex.go b/controllers/regex.go index ae913323c..2c1f0c51e 100644 --- a/controllers/regex.go +++ b/controllers/regex.go @@ -22,7 +22,7 @@ func validName(name string) bool { if !reg.MatchString(name) { return false } - if len(name) > 15 { + if len(name) < 5 || len(name) > 32 { return false } return true diff --git a/controllers/regex_test.go b/controllers/regex_test.go index 3690d1c9c..e2ab1c0b9 100644 --- a/controllers/regex_test.go +++ b/controllers/regex_test.go @@ -27,19 +27,33 @@ func TestValidName(t *testing.T) { Want: false, }, { - Name: "nametoolong", + Name: "longname", Args: args{ Name: "TestvalidNameTestvalidName", }, - Want: false, + Want: true, }, { - Name: "maxlength", + Name: "max length", Args: args{ Name: "123456789012345", }, Want: true, }, + { + Name: "min length", + Args: args{ + Name: "ama", + }, + Want: false, + }, + { + Name: "toolong", + Args: args{ + Name: "123456789012345123123123123123123123123123123", + }, + Want: false, + }, } for _, tt := range tests { t.Run(tt.Name, func(t *testing.T) { From 069226ad6d97c0777da5eb6fde14bd347dbc4c6c Mon Sep 17 00:00:00 2001 From: Gabriel de Souza Seibel Date: Tue, 8 Aug 2023 07:29:38 -0300 Subject: [PATCH 18/33] [NET-477] Pick AMB URL dynamically (#2489) * Introduce config for environment * Introduce func to get environment * Choose accounts api host from environment * Test the ee package on workflows * Use build tag ee for license_test.go --- .github/workflows/test.yml | 1 + config/config.go | 1 + ee/license.go | 13 ++++++- ee/license_test.go | 77 ++++++++++++++++++++++++++++++++++++++ ee/types.go | 15 +++++++- servercfg/serverconf.go | 11 ++++++ 6 files changed, 115 insertions(+), 3 deletions(-) create mode 100644 ee/license_test.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 08bd67a5a..649a63137 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -51,6 +51,7 @@ jobs: run: | go vet ./... go test -p 1 ./... -v + go test -p 1 ./ee -v --tags ee env: DATABASE: sqlite CLIENT_MODE: "off" diff --git a/config/config.go b/config/config.go index fa47fab66..934837a38 100644 --- a/config/config.go +++ b/config/config.go @@ -86,6 +86,7 @@ type ServerConfig struct { NetworksLimit int `yaml:"network_limit"` HostsLimit int `yaml:"host_limit"` DeployedByOperator bool `yaml:"deployed_by_operator"` + Environment string `yaml:"environment"` } // SQLConfig - Generic SQL Config diff --git a/ee/license.go b/ee/license.go index 5f05df432..aeb9b3f7f 100644 --- a/ee/license.go +++ b/ee/license.go @@ -198,7 +198,7 @@ func validateLicenseKey(encryptedData []byte, publicKey *[32]byte) ([]byte, erro return nil, err } - req, err := http.NewRequest(http.MethodPost, api_endpoint, bytes.NewReader(requestBody)) + req, err := http.NewRequest(http.MethodPost, getAccountsHost()+"/api/v1/license/validate", bytes.NewReader(requestBody)) if err != nil { return nil, err } @@ -229,6 +229,17 @@ func validateLicenseKey(encryptedData []byte, publicKey *[32]byte) ([]byte, erro return body, err } +func getAccountsHost() string { + switch servercfg.GetEnvironment() { + case "dev": + return accountsHostDevelopment + case "staging": + return accountsHostStaging + default: + return accountsHostProduction + } +} + func cacheResponse(response []byte) error { var lrc = licenseResponseCache{ Body: response, diff --git a/ee/license_test.go b/ee/license_test.go new file mode 100644 index 000000000..30984680d --- /dev/null +++ b/ee/license_test.go @@ -0,0 +1,77 @@ +//go:build ee +// +build ee + +package ee + +import ( + "github.com/gravitl/netmaker/config" + "testing" +) + +func Test_getAccountsHost(t *testing.T) { + tests := []struct { + name string + envK string + envV string + conf string + want string + }{ + { + name: "no env var and no conf", + envK: "NOT_THE_CORRECT_ENV_VAR", + envV: "dev", + want: "https://api.accounts.netmaker.io", + }, + { + name: "dev env var", + envK: "ENVIRONMENT", + envV: "dev", + want: "https://api.dev.accounts.netmaker.io", + }, + { + name: "staging env var", + envK: "ENVIRONMENT", + envV: "staging", + want: "https://api.staging.accounts.netmaker.io", + }, + { + name: "prod env var", + envK: "ENVIRONMENT", + envV: "prod", + want: "https://api.accounts.netmaker.io", + }, + { + name: "dev conf", + conf: "dev", + want: "https://api.dev.accounts.netmaker.io", + }, + { + name: "staging conf", + conf: "staging", + want: "https://api.staging.accounts.netmaker.io", + }, + { + name: "prod conf", + conf: "prod", + want: "https://api.accounts.netmaker.io", + }, + { + name: "env var vs conf precedence", + envK: "ENVIRONMENT", + envV: "prod", + conf: "staging", + want: "https://api.accounts.netmaker.io", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + config.Config.Server.Environment = tt.conf + if tt.envK != "" { + t.Setenv(tt.envK, tt.envV) + } + if got := getAccountsHost(); got != tt.want { + t.Errorf("getAccountsHost() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/ee/types.go b/ee/types.go index 17b5dd331..e7f77464e 100644 --- a/ee/types.go +++ b/ee/types.go @@ -1,9 +1,20 @@ package ee -import "fmt" +import ( + "fmt" +) + +// constants for accounts api hosts +const ( + // accountsHostDevelopment is the accounts api host for development environment + accountsHostDevelopment = "https://api.dev.accounts.netmaker.io" + // accountsHostStaging is the accounts api host for staging environment + accountsHostStaging = "https://api.staging.accounts.netmaker.io" + // accountsHostProduction is the accounts api host for production environment + accountsHostProduction = "https://api.accounts.netmaker.io" +) const ( - api_endpoint = "https://api.accounts.netmaker.io/api/v1/license/validate" license_cache_key = "license_response_cache" license_validation_err_msg = "invalid license" server_id_key = "nm-server-id" diff --git a/servercfg/serverconf.go b/servercfg/serverconf.go index f4e70030e..9767690e3 100644 --- a/servercfg/serverconf.go +++ b/servercfg/serverconf.go @@ -783,6 +783,17 @@ func DeployedByOperator() bool { return config.Config.Server.DeployedByOperator } +// GetEnvironment returns the environment the server is running in (e.g. dev, staging, prod...) +func GetEnvironment() string { + if env := os.Getenv("ENVIRONMENT"); env != "" { + return env + } + if env := config.Config.Server.Environment; env != "" { + return env + } + return "" +} + // parseStunList - turn string into slice of StunServers func parseStunList(stunString string) ([]models.StunServer, error) { var err error From 100e2fd6f407d57faf341cc608bab244de164074 Mon Sep 17 00:00:00 2001 From: bornav <51048565+bornav@users.noreply.github.com> Date: Tue, 8 Aug 2023 12:29:55 +0200 Subject: [PATCH 19/33] [Feature]: nm-quick script tackling arm TODO support (#2488) * domain flag for auto installs * use static servers with custom domain (#2421) * send delete peer update always * fix add/remove host api calls * keep mq updates in a single go func * move branch test logic to devops (#2443) * handle IOT OS * save server name to env (#2460) * ensure branch test servers available after test runs (#2467) * save server name to env * free server always; add PR to discord messages * use correct method to delete droplets (#2468) * quick fix for the launcher * removed exit when triggering not supported exit and removed the TODO comments related to this issue --------- Co-authored-by: Matthew R Kasun Co-authored-by: Alex Feiszli <31018251+afeiszli@users.noreply.github.com> Co-authored-by: Christopher Blaha Co-authored-by: Abhishek Kondur Co-authored-by: Abhishek K <32607604+abhishek9686@users.noreply.github.com> --- .github/workflows/deletedroplets.yml | 22 ++++++++++++++++++---- scripts/nm-quick.sh | 17 ++++++++++++----- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/.github/workflows/deletedroplets.yml b/.github/workflows/deletedroplets.yml index 359bd7b0d..5e1917929 100644 --- a/.github/workflows/deletedroplets.yml +++ b/.github/workflows/deletedroplets.yml @@ -18,7 +18,13 @@ jobs: if_no_artifact_found: warn - name: get server name run: | - echo "SERVER=$(cat ./server/server) >> $GITHUB_ENV" + echo "SERVER=$(cat ./server/server)" >> $GITHUB_ENV + - name: get PR + env: + GH_TOKEN: ${{ github.token }} + run: | + echo "pull_request=$(gh api -H 'Accept: application/vnd.github+json' -H 'X-Github-Api-Version: 2022-11-28' \ + /repos/${{ github.repository}}/actions/runs/${{ github.event.workflow_run.id }} | jq .pull_requests[0].number)" >> $GITHUB_ENV - name: discord success message uses: appleboy/discord-action@master with: @@ -32,7 +38,7 @@ jobs: if: success() || failure() run: | sleep 15m - curl -X GET \ + curl -X DELETE \ -H "Content-Type: application/json" \ -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" \ "https://api.digitalocean.com/v2/droplets?tag_name=$TAG" @@ -40,6 +46,7 @@ jobs: DIGITALOCEAN_TOKEN: ${{ secrets.DIGITALOCEAN_TOKEN }} TAG: ${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }} - name: mark server as available + if: success() || failure() uses: appleboy/ssh-action@master with: host: server.${{ env.SERVER }}.clustercat.com @@ -59,7 +66,13 @@ jobs: if_no_artifact_found: warn - name: get server name run: | - echo "SERVER=$(cat ./server/server) >> $GITHUB_ENV" + echo "SERVER=$(cat ./server/server)" >> $GITHUB_ENV + - name: get PR + env: + GH_TOKEN: ${{ github.token }} + run: | + echo "pull_request=$(gh api -H 'Accept: application/vnd.github+json' -H 'X-Github-Api-Version: 2022-11-28' \ + /repos/${{ github.repository}}/actions/runs/${{ github.event.workflow_run.id }} | jq .pull_requests[0].number)" >> $GITHUB_ENV - name: discord failure message uses: appleboy/discord-action@master with: @@ -82,7 +95,7 @@ jobs: if: success() || failure() run: | sleep 3h - curl -X GET \ + curl -X DELETE \ -H "Content-Type: application/json" \ -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" \ "https://api.digitalocean.com/v2/droplets?tag_name=$TAG" @@ -90,6 +103,7 @@ jobs: DIGITALOCEAN_TOKEN: ${{ secrets.DIGITALOCEAN_TOKEN }} TAG: ${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }} - name: mark server as available + if: success() || failure() uses: appleboy/ssh-action@master with: host: server.${{ env.SERVER }}.clustercat.com diff --git a/scripts/nm-quick.sh b/scripts/nm-quick.sh index 3f5339fda..7c4e42740 100755 --- a/scripts/nm-quick.sh +++ b/scripts/nm-quick.sh @@ -176,8 +176,7 @@ setup_netclient() { netclient uninstall set -e - # TODO arm support - wget -qO netclient https://github.com/gravitl/netclient/releases/download/$LATEST/netclient-linux-amd64 + wget -qO netclient https://github.com/gravitl/netclient/releases/download/$LATEST/netclient-linux-$ARCH chmod +x netclient ./netclient install echo "Register token: $TOKEN" @@ -227,8 +226,7 @@ configure_netclient() { # setup_nmctl - pulls nmctl and makes it executable setup_nmctl() { - # TODO arm support - local URL="https://github.com/gravitl/netmaker/releases/download/$LATEST/nmctl-linux-amd64" + local URL="https://github.com/gravitl/netmaker/releases/download/$LATEST/nmctl-linux-$ARCH" echo "Downloading nmctl..." wget -qO /usr/bin/nmctl "$URL" @@ -430,7 +428,16 @@ install_dependencies() { # ask the user to install manually and continue when ready exit 1 fi - + # TODO add other supported architectures + ARCH=$(uname -m) + if [ "$ARCH" = "x86_64" ]; then + ARCH=amd64 + elif [ "$ARCH" = "aarch64" ]; then + ARCH=arm64 + else + echo "Unsupported architechure" + # exit 1 + fi set -- $dependencies ${update_cmd} From db1a20f21ce855956bb16109a91459db8947748f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Aug 2023 23:13:49 +0530 Subject: [PATCH 20/33] Bump github.com/go-playground/validator/v10 from 10.14.1 to 10.15.0 (#2503) Bumps [github.com/go-playground/validator/v10](https://github.com/go-playground/validator) from 10.14.1 to 10.15.0. - [Release notes](https://github.com/go-playground/validator/releases) - [Commits](https://github.com/go-playground/validator/compare/v10.14.1...v10.15.0) --- updated-dependencies: - dependency-name: github.com/go-playground/validator/v10 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4c5524d08..bf8afa251 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.19 require ( github.com/eclipse/paho.mqtt.golang v1.4.3 - github.com/go-playground/validator/v10 v10.14.1 + github.com/go-playground/validator/v10 v10.15.0 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/uuid v1.3.0 github.com/gorilla/handlers v1.5.1 diff --git a/go.sum b/go.sum index d71d2c468..a95179c3e 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,8 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.14.1 h1:9c50NUPC30zyuKprjL3vNZ0m5oG+jU0zvx4AqHGnv4k= -github.com/go-playground/validator/v10 v10.14.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/go-playground/validator/v10 v10.15.0 h1:nDU5XeOKtB3GEa+uB7GNYwhVKsgjAR7VgKoNB6ryXfw= +github.com/go-playground/validator/v10 v10.15.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= From 21ea9200f8be3af8d5d8e7adaca835275c70d14e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Aug 2023 23:14:22 +0530 Subject: [PATCH 21/33] Bump golang.org/x/oauth2 from 0.10.0 to 0.11.0 (#2502) Bumps [golang.org/x/oauth2](https://github.com/golang/oauth2) from 0.10.0 to 0.11.0. - [Commits](https://github.com/golang/oauth2/compare/v0.10.0...v0.11.0) --- updated-dependencies: - dependency-name: golang.org/x/oauth2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index bf8afa251..f73dba38c 100644 --- a/go.mod +++ b/go.mod @@ -15,11 +15,11 @@ require ( github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/stretchr/testify v1.8.4 github.com/txn2/txeh v1.4.0 - golang.org/x/crypto v0.11.0 - golang.org/x/net v0.12.0 // indirect - golang.org/x/oauth2 v0.10.0 - golang.org/x/sys v0.10.0 // indirect - golang.org/x/text v0.11.0 // indirect + golang.org/x/crypto v0.12.0 + golang.org/x/net v0.14.0 // indirect + golang.org/x/oauth2 v0.11.0 + golang.org/x/sys v0.11.0 // indirect + golang.org/x/text v0.12.0 // indirect golang.zx2c4.com/wireguard/wgctrl v0.0.0-20220324164955-056925b7df31 google.golang.org/protobuf v1.31.0 // indirect gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index a95179c3e..537e31453 100644 --- a/go.sum +++ b/go.sum @@ -112,8 +112,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220208050332-20e1d8d225ab/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -123,10 +123,10 @@ golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211111083644-e5c967477495/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= -golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= +golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -141,8 +141,8 @@ golang.org/x/sys v0.0.0-20211110154304-99a53858aa08/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220207234003-57398862261d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -150,8 +150,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From ab1bffaf3b4708a72abd18f1e7baacc6d21e6b5c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Aug 2023 23:15:37 +0530 Subject: [PATCH 22/33] Bump alpine from 3.18.2 to 3.18.3 (#2500) Bumps alpine from 3.18.2 to 3.18.3. --- updated-dependencies: - dependency-name: alpine dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- Dockerfile-quick | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e2d5a3c3f..1c5cefaad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ COPY . . RUN GOOS=linux CGO_ENABLED=1 go build -ldflags="-s -w " -tags ${tags} . # RUN go build -tags=ee . -o netmaker main.go -FROM alpine:3.18.2 +FROM alpine:3.18.3 # add a c lib # set the working directory diff --git a/Dockerfile-quick b/Dockerfile-quick index feb67a939..9502f335c 100644 --- a/Dockerfile-quick +++ b/Dockerfile-quick @@ -1,5 +1,5 @@ #first stage - builder -FROM alpine:3.18.2 +FROM alpine:3.18.3 ARG version WORKDIR /app COPY ./netmaker /root/netmaker From 7cb526c3156ede580e99ebc8112489a669b35c17 Mon Sep 17 00:00:00 2001 From: Farukh Khan Date: Wed, 9 Aug 2023 01:47:22 +0800 Subject: [PATCH 23/33] Net 509 515 (#2496) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * NET-509 * External client config files with IPv6 endpoints now have the [] separating the address and port. * NET-515 * Increased network name max length to 32 * NET-509-515 * Updated unit test for network name max length check. * Updated extclient endpoint ip string manipulation to use sprintf * Added proper error message for network name length more than max allowed. * NET-515 small typo fix for error strings should not be capitalized --- controllers/ext_client.go | 7 ++++++- controllers/network.go | 8 ++++++++ controllers/network_test.go | 4 ++-- models/network.go | 8 ++++---- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/controllers/ext_client.go b/controllers/ext_client.go index 4155c6537..c372064cb 100644 --- a/controllers/ext_client.go +++ b/controllers/ext_client.go @@ -217,7 +217,12 @@ func getExtClientConf(w http.ResponseWriter, r *http.Request) { if network.DefaultKeepalive != 0 { keepalive = "PersistentKeepalive = " + strconv.Itoa(int(network.DefaultKeepalive)) } - gwendpoint := host.EndpointIP.String() + ":" + strconv.Itoa(host.ListenPort) + gwendpoint := "" + if host.EndpointIP.To4() == nil { + gwendpoint = fmt.Sprintf("[%s]:%d", host.EndpointIP.String(), host.ListenPort) + } else { + gwendpoint = fmt.Sprintf("%s:%d", host.EndpointIP.String(), host.ListenPort) + } newAllowedIPs := network.AddressRange if newAllowedIPs != "" && network.AddressRange6 != "" { newAllowedIPs += "," diff --git a/controllers/network.go b/controllers/network.go index 11541a76e..8558b13f4 100644 --- a/controllers/network.go +++ b/controllers/network.go @@ -245,6 +245,14 @@ func createNetwork(w http.ResponseWriter, r *http.Request) { return } + if len(network.NetID) > 32 { + err := errors.New("network name shouldn't exceed 32 characters") + logger.Log(0, r.Header.Get("user"), "failed to create network: ", + err.Error()) + logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest")) + return + } + if network.AddressRange == "" && network.AddressRange6 == "" { err := errors.New("IPv4 or IPv6 CIDR required") logger.Log(0, r.Header.Get("user"), "failed to create network: ", diff --git a/controllers/network_test.go b/controllers/network_test.go index 17192d54c..ab7a476c6 100644 --- a/controllers/network_test.go +++ b/controllers/network_test.go @@ -80,7 +80,7 @@ func TestDeleteNetwork(t *testing.T) { err := logic.DeleteNetwork("skynet") assert.Nil(t, err) }) - t.Run("NonExistantNetwork", func(t *testing.T) { + t.Run("NonExistentNetwork", func(t *testing.T) { err := logic.DeleteNetwork("skynet") assert.Nil(t, err) }) @@ -151,7 +151,7 @@ func TestValidateNetwork(t *testing.T) { { testname: "NetIDTooLong", network: models.Network{ - NetID: "LongNetIDName", + NetID: "LongNetIDNameForMaxCharactersTest", }, errMessage: "Field validation for 'NetID' failed on the 'max' tag", }, diff --git a/models/network.go b/models/network.go index 7c02c8b7b..b075a3ef8 100644 --- a/models/network.go +++ b/models/network.go @@ -11,10 +11,10 @@ import ( type Network struct { AddressRange string `json:"addressrange" bson:"addressrange" validate:"omitempty,cidrv4"` AddressRange6 string `json:"addressrange6" bson:"addressrange6" validate:"omitempty,cidrv6"` - NetID string `json:"netid" bson:"netid" validate:"required,min=1,max=12,netid_valid"` + NetID string `json:"netid" bson:"netid" validate:"required,min=1,max=32,netid_valid"` NodesLastModified int64 `json:"nodeslastmodified" bson:"nodeslastmodified"` NetworkLastModified int64 `json:"networklastmodified" bson:"networklastmodified"` - DefaultInterface string `json:"defaultinterface" bson:"defaultinterface" validate:"min=1,max=15"` + DefaultInterface string `json:"defaultinterface" bson:"defaultinterface" validate:"min=1,max=35"` DefaultListenPort int32 `json:"defaultlistenport,omitempty" bson:"defaultlistenport,omitempty" validate:"omitempty,min=1024,max=65535"` NodeLimit int32 `json:"nodelimit" bson:"nodelimit"` DefaultPostDown string `json:"defaultpostdown" bson:"defaultpostdown"` @@ -30,7 +30,7 @@ type Network struct { // SaveData - sensitive fields of a network that should be kept the same type SaveData struct { // put sensitive fields here - NetID string `json:"netid" bson:"netid" validate:"required,min=1,max=12,netid_valid"` + NetID string `json:"netid" bson:"netid" validate:"required,min=1,max=32,netid_valid"` } // Network.SetNodesLastModified - sets nodes last modified on network, depricated @@ -49,7 +49,7 @@ func (network *Network) SetDefaults() { network.DefaultUDPHolePunch = "no" } if network.DefaultInterface == "" { - if len(network.NetID) < 13 { + if len(network.NetID) < 33 { network.DefaultInterface = "nm-" + network.NetID } else { network.DefaultInterface = network.NetID From 3c86c9b88962309258474a6e1cb13d264f4d0e20 Mon Sep 17 00:00:00 2001 From: Gabriel de Souza Seibel Date: Tue, 8 Aug 2023 14:47:49 -0300 Subject: [PATCH 24/33] [NET-494 / ACC-322] New free tier limits (#2495) * Rename var * Rename consts and use iota * Use switch instead of repeated else if * Rename limits related vars * Introduce new free tier limits * Measure new limits and report on license validation * Separate usage and limits, have new ones * Don't check for hosts and clients limits, but for machines instead * Error on egress creation @ free tier w/ internet gateways * Remove clients and hosts limit from code * Rename var * Rename consts and use iota * Use switch instead of repeated else if * Rename limits related vars * Introduce new free tier limits * Measure new limits and report on license validation * Separate usage and limits, have new ones * Don't check for hosts and clients limits, but for machines instead * Error on egress creation @ free tier w/ internet gateways * Remove clients and hosts limit from code --- config/config.go | 5 ++-- controllers/ext_client.go | 2 +- controllers/limits.go | 56 ++++++++++++++++++++++++++++----------- controllers/network.go | 2 +- controllers/node.go | 4 +-- controllers/server.go | 20 +++++++++++--- controllers/user.go | 2 +- ee/license.go | 2 +- ee/types.go | 46 ++++++++++++++++++-------------- ee/util.go | 19 +++++++++---- logic/gateway.go | 48 ++++++++++++++++++++++++--------- logic/hosts.go | 14 +++++----- logic/serverconf.go | 35 +++++++++++++----------- servercfg/serverconf.go | 34 +++++++++++++----------- 14 files changed, 185 insertions(+), 104 deletions(-) diff --git a/config/config.go b/config/config.go index 934837a38..569d430af 100644 --- a/config/config.go +++ b/config/config.go @@ -82,9 +82,10 @@ type ServerConfig struct { TurnPassword string `yaml:"turn_password"` UseTurn bool `yaml:"use_turn"` UsersLimit int `yaml:"user_limit"` - ClientsLimit int `yaml:"client_limit"` NetworksLimit int `yaml:"network_limit"` - HostsLimit int `yaml:"host_limit"` + MachinesLimit int `yaml:"machines_limit"` + IngressesLimit int `yaml:"ingresses_limit"` + EgressesLimit int `yaml:"egresses_limit"` DeployedByOperator bool `yaml:"deployed_by_operator"` Environment string `yaml:"environment"` } diff --git a/controllers/ext_client.go b/controllers/ext_client.go index c372064cb..35d6358ae 100644 --- a/controllers/ext_client.go +++ b/controllers/ext_client.go @@ -29,7 +29,7 @@ func extClientHandlers(r *mux.Router) { r.HandleFunc("/api/extclients/{network}/{clientid}/{type}", logic.NetUserSecurityCheck(false, true, http.HandlerFunc(getExtClientConf))).Methods(http.MethodGet) r.HandleFunc("/api/extclients/{network}/{clientid}", logic.NetUserSecurityCheck(false, true, http.HandlerFunc(updateExtClient))).Methods(http.MethodPut) r.HandleFunc("/api/extclients/{network}/{clientid}", logic.NetUserSecurityCheck(false, true, http.HandlerFunc(deleteExtClient))).Methods(http.MethodDelete) - r.HandleFunc("/api/extclients/{network}/{nodeid}", logic.NetUserSecurityCheck(false, true, checkFreeTierLimits(clients_l, http.HandlerFunc(createExtClient)))).Methods(http.MethodPost) + r.HandleFunc("/api/extclients/{network}/{nodeid}", logic.NetUserSecurityCheck(false, true, checkFreeTierLimits(limitChoiceMachines, http.HandlerFunc(createExtClient)))).Methods(http.MethodPost) } func checkIngressExists(nodeID string) bool { diff --git a/controllers/limits.go b/controllers/limits.go index 2b4cee159..e3a7b7b16 100644 --- a/controllers/limits.go +++ b/controllers/limits.go @@ -10,36 +10,60 @@ import ( // limit consts const ( - node_l = 0 - networks_l = 1 - users_l = 2 - clients_l = 3 + limitChoiceNetworks = iota + limitChoiceUsers + limitChoiceMachines + limitChoiceIngress + limitChoiceEgress ) -func checkFreeTierLimits(limit_choice int, next http.Handler) http.HandlerFunc { +func checkFreeTierLimits(limitChoice int, next http.Handler) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var errorResponse = models.ErrorResponse{ - Code: http.StatusForbidden, Message: "free tier limits exceeded on networks", + Code: http.StatusForbidden, Message: "free tier limits exceeded on ", } - if logic.Free_Tier { // check that free tier limits not exceeded - if limit_choice == networks_l { + if logic.FreeTier { // check that free tier limits not exceeded + switch limitChoice { + case limitChoiceNetworks: currentNetworks, err := logic.GetNetworks() - if (err != nil && !database.IsEmptyRecord(err)) || len(currentNetworks) >= logic.Networks_Limit { + if (err != nil && !database.IsEmptyRecord(err)) || + len(currentNetworks) >= logic.NetworksLimit { + errorResponse.Message += "networks" logic.ReturnErrorResponse(w, r, errorResponse) return } - } else if limit_choice == users_l { + case limitChoiceUsers: users, err := logic.GetUsers() - if (err != nil && !database.IsEmptyRecord(err)) || len(users) >= logic.Users_Limit { - errorResponse.Message = "free tier limits exceeded on users" + if (err != nil && !database.IsEmptyRecord(err)) || + len(users) >= logic.UsersLimit { + errorResponse.Message += "users" logic.ReturnErrorResponse(w, r, errorResponse) return } - } else if limit_choice == clients_l { - clients, err := logic.GetAllExtClients() - if (err != nil && !database.IsEmptyRecord(err)) || len(clients) >= logic.Clients_Limit { - errorResponse.Message = "free tier limits exceeded on external clients" + case limitChoiceMachines: + hosts, hErr := logic.GetAllHosts() + clients, cErr := logic.GetAllExtClients() + if (hErr != nil && !database.IsEmptyRecord(hErr)) || + (cErr != nil && !database.IsEmptyRecord(cErr)) || + len(hosts)+len(clients) >= logic.MachinesLimit { + errorResponse.Message += "machines" + logic.ReturnErrorResponse(w, r, errorResponse) + return + } + case limitChoiceIngress: + ingresses, err := logic.GetAllIngresses() + if (err != nil && !database.IsEmptyRecord(err)) || + len(ingresses) >= logic.IngressesLimit { + errorResponse.Message += "ingresses" + logic.ReturnErrorResponse(w, r, errorResponse) + return + } + case limitChoiceEgress: + egresses, err := logic.GetAllEgresses() + if (err != nil && !database.IsEmptyRecord(err)) || + len(egresses) >= logic.EgressesLimit { + errorResponse.Message += "egresses" logic.ReturnErrorResponse(w, r, errorResponse) return } diff --git a/controllers/network.go b/controllers/network.go index 8558b13f4..b3e799b71 100644 --- a/controllers/network.go +++ b/controllers/network.go @@ -21,7 +21,7 @@ import ( func networkHandlers(r *mux.Router) { r.HandleFunc("/api/networks", logic.SecurityCheck(false, http.HandlerFunc(getNetworks))).Methods(http.MethodGet) - r.HandleFunc("/api/networks", logic.SecurityCheck(true, checkFreeTierLimits(networks_l, http.HandlerFunc(createNetwork)))).Methods(http.MethodPost) + r.HandleFunc("/api/networks", logic.SecurityCheck(true, checkFreeTierLimits(limitChoiceNetworks, http.HandlerFunc(createNetwork)))).Methods(http.MethodPost) r.HandleFunc("/api/networks/{networkname}", logic.SecurityCheck(false, http.HandlerFunc(getNetwork))).Methods(http.MethodGet) r.HandleFunc("/api/networks/{networkname}", logic.SecurityCheck(true, http.HandlerFunc(deleteNetwork))).Methods(http.MethodDelete) r.HandleFunc("/api/networks/{networkname}", logic.SecurityCheck(true, http.HandlerFunc(updateNetwork))).Methods(http.MethodPut) diff --git a/controllers/node.go b/controllers/node.go index 64055ebd4..28de485d1 100644 --- a/controllers/node.go +++ b/controllers/node.go @@ -28,9 +28,9 @@ func nodeHandlers(r *mux.Router) { r.HandleFunc("/api/nodes/{network}/{nodeid}", Authorize(true, true, "node", http.HandlerFunc(getNode))).Methods(http.MethodGet) r.HandleFunc("/api/nodes/{network}/{nodeid}", Authorize(false, true, "node", http.HandlerFunc(updateNode))).Methods(http.MethodPut) r.HandleFunc("/api/nodes/{network}/{nodeid}", Authorize(true, true, "node", http.HandlerFunc(deleteNode))).Methods(http.MethodDelete) - r.HandleFunc("/api/nodes/{network}/{nodeid}/creategateway", Authorize(false, true, "user", http.HandlerFunc(createEgressGateway))).Methods(http.MethodPost) + r.HandleFunc("/api/nodes/{network}/{nodeid}/creategateway", Authorize(false, true, "user", checkFreeTierLimits(limitChoiceEgress, http.HandlerFunc(createEgressGateway)))).Methods(http.MethodPost) r.HandleFunc("/api/nodes/{network}/{nodeid}/deletegateway", Authorize(false, true, "user", http.HandlerFunc(deleteEgressGateway))).Methods(http.MethodDelete) - r.HandleFunc("/api/nodes/{network}/{nodeid}/createingress", logic.SecurityCheck(false, http.HandlerFunc(createIngressGateway))).Methods(http.MethodPost) + r.HandleFunc("/api/nodes/{network}/{nodeid}/createingress", logic.SecurityCheck(false, checkFreeTierLimits(limitChoiceIngress, http.HandlerFunc(createIngressGateway)))).Methods(http.MethodPost) r.HandleFunc("/api/nodes/{network}/{nodeid}/deleteingress", logic.SecurityCheck(false, http.HandlerFunc(deleteIngressGateway))).Methods(http.MethodDelete) r.HandleFunc("/api/nodes/{network}/{nodeid}", Authorize(true, true, "node", http.HandlerFunc(updateNode))).Methods(http.MethodPost) r.HandleFunc("/api/nodes/adm/{network}/authenticate", authenticate).Methods(http.MethodPost) diff --git a/controllers/server.go b/controllers/server.go index 79c8e21f0..df23f1229 100644 --- a/controllers/server.go +++ b/controllers/server.go @@ -24,12 +24,16 @@ func serverHandlers(r *mux.Router) { r.HandleFunc("/api/server/status", http.HandlerFunc(getStatus)).Methods(http.MethodGet) r.HandleFunc("/api/server/usage", Authorize(true, false, "user", http.HandlerFunc(getUsage))).Methods(http.MethodGet) } + +// TODO move to EE package? there is a function and a type there for that already func getUsage(w http.ResponseWriter, r *http.Request) { type usage struct { - Hosts int `json:"hosts"` - Clients int `json:"clients"` - Networks int `json:"networks"` - Users int `json:"users"` + Hosts int `json:"hosts"` + Clients int `json:"clients"` + Networks int `json:"networks"` + Users int `json:"users"` + Ingresses int `json:"ingresses"` + Egresses int `json:"egresses"` } var serverUsage usage hosts, err := logic.GetAllHosts() @@ -48,6 +52,14 @@ func getUsage(w http.ResponseWriter, r *http.Request) { if err == nil { serverUsage.Networks = len(networks) } + ingresses, err := logic.GetAllIngresses() + if err == nil { + serverUsage.Ingresses = len(ingresses) + } + egresses, err := logic.GetAllEgresses() + if err == nil { + serverUsage.Egresses = len(egresses) + } w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(models.SuccessResponse{ Code: http.StatusOK, diff --git a/controllers/user.go b/controllers/user.go index fdd59fca5..0a7229c17 100644 --- a/controllers/user.go +++ b/controllers/user.go @@ -30,7 +30,7 @@ func userHandlers(r *mux.Router) { r.HandleFunc("/api/users/{username}", logic.SecurityCheck(false, logic.ContinueIfUserMatch(http.HandlerFunc(updateUser)))).Methods(http.MethodPut) r.HandleFunc("/api/users/networks/{username}", logic.SecurityCheck(true, http.HandlerFunc(updateUserNetworks))).Methods(http.MethodPut) r.HandleFunc("/api/users/{username}/adm", logic.SecurityCheck(true, http.HandlerFunc(updateUserAdm))).Methods(http.MethodPut) - r.HandleFunc("/api/users/{username}", logic.SecurityCheck(true, checkFreeTierLimits(users_l, http.HandlerFunc(createUser)))).Methods(http.MethodPost) + r.HandleFunc("/api/users/{username}", logic.SecurityCheck(true, checkFreeTierLimits(limitChoiceUsers, http.HandlerFunc(createUser)))).Methods(http.MethodPost) r.HandleFunc("/api/users/{username}", logic.SecurityCheck(true, http.HandlerFunc(deleteUser))).Methods(http.MethodDelete) r.HandleFunc("/api/users/{username}", logic.SecurityCheck(false, logic.ContinueIfUserMatch(http.HandlerFunc(getUser)))).Methods(http.MethodGet) r.HandleFunc("/api/users", logic.SecurityCheck(true, http.HandlerFunc(getUsers))).Methods(http.MethodGet) diff --git a/ee/license.go b/ee/license.go index aeb9b3f7f..8bf7345ba 100644 --- a/ee/license.go +++ b/ee/license.go @@ -81,7 +81,7 @@ func ValidateLicense() (err error) { licenseSecret := LicenseSecret{ AssociatedID: netmakerTenantID, - Limits: getCurrentServerLimit(), + Usage: getCurrentServerUsage(), } secretData, err := json.Marshal(&licenseSecret) diff --git a/ee/types.go b/ee/types.go index e7f77464e..355bc6d3a 100644 --- a/ee/types.go +++ b/ee/types.go @@ -24,15 +24,17 @@ var errValidation = fmt.Errorf(license_validation_err_msg) // LicenseKey - the license key struct representation with associated data type LicenseKey struct { - LicenseValue string `json:"license_value"` // actual (public) key and the unique value for the key - Expiration int64 `json:"expiration"` - LimitServers int `json:"limit_servers"` - LimitUsers int `json:"limit_users"` - LimitHosts int `json:"limit_hosts"` - LimitNetworks int `json:"limit_networks"` - LimitClients int `json:"limit_clients"` - Metadata string `json:"metadata"` - IsActive bool `json:"is_active"` // yes if active + LicenseValue string `json:"license_value"` // actual (public) key and the unique value for the key + Expiration int64 `json:"expiration"` + UsageServers int `json:"usage_servers"` + UsageUsers int `json:"usage_users"` + UsageClients int `json:"usage_clients"` + UsageHosts int `json:"usage_hosts"` + UsageNetworks int `json:"usage_networks"` + UsageIngresses int `json:"usage_ingresses"` + UsageEgresses int `json:"usage_egresses"` + Metadata string `json:"metadata"` + IsActive bool `json:"is_active"` // yes if active } // ValidatedLicense - the validated license struct @@ -43,26 +45,30 @@ type ValidatedLicense struct { // LicenseSecret - the encrypted struct for sending user-id type LicenseSecret struct { - AssociatedID string `json:"associated_id" binding:"required"` // UUID for user foreign key to User table - Limits LicenseLimits `json:"limits" binding:"required"` + AssociatedID string `json:"associated_id" binding:"required"` // UUID for user foreign key to User table + Usage Usage `json:"usage" binding:"required"` } -// LicenseLimits - struct license limits -type LicenseLimits struct { - Servers int `json:"servers"` - Users int `json:"users"` - Hosts int `json:"hosts"` - Clients int `json:"clients"` - Networks int `json:"networks"` +// Usage - struct for license usage +type Usage struct { + Servers int `json:"servers"` + Users int `json:"users"` + Hosts int `json:"hosts"` + Clients int `json:"clients"` + Networks int `json:"networks"` + Ingresses int `json:"ingresses"` + Egresses int `json:"egresses"` } -// LicenseLimits.SetDefaults - sets the default values for limits -func (l *LicenseLimits) SetDefaults() { +// Usage.SetDefaults - sets the default values for usage +func (l *Usage) SetDefaults() { l.Clients = 0 l.Servers = 1 l.Hosts = 0 l.Users = 1 l.Networks = 0 + l.Ingresses = 0 + l.Egresses = 0 } // ValidateLicenseRequest - used for request to validate license endpoint diff --git a/ee/util.go b/ee/util.go index b63cf1582..8712981f9 100644 --- a/ee/util.go +++ b/ee/util.go @@ -30,14 +30,15 @@ func base64decode(input string) []byte { return bytes } -func getCurrentServerLimit() (limits LicenseLimits) { + +func getCurrentServerUsage() (limits Usage) { limits.SetDefaults() - hosts, err := logic.GetAllHosts() - if err == nil { + hosts, hErr := logic.GetAllHosts() + if hErr == nil { limits.Hosts = len(hosts) } - clients, err := logic.GetAllExtClients() - if err == nil { + clients, cErr := logic.GetAllExtClients() + if cErr == nil { limits.Clients = len(clients) } users, err := logic.GetUsers() @@ -48,5 +49,13 @@ func getCurrentServerLimit() (limits LicenseLimits) { if err == nil { limits.Networks = len(networks) } + ingresses, err := logic.GetAllIngresses() + if err == nil { + limits.Ingresses = len(ingresses) + } + egresses, err := logic.GetAllEgresses() + if err == nil { + limits.Egresses = len(egresses) + } return } diff --git a/logic/gateway.go b/logic/gateway.go index 90827e13f..094ffa800 100644 --- a/logic/gateway.go +++ b/logic/gateway.go @@ -11,6 +11,36 @@ import ( "github.com/gravitl/netmaker/servercfg" ) +// GetAllIngresses - gets all the hosts that are ingresses +func GetAllIngresses() ([]models.Node, error) { + nodes, err := GetAllNodes() + if err != nil { + return nil, err + } + ingresses := make([]models.Node, 0) + for _, node := range nodes { + if node.IsIngressGateway { + ingresses = append(ingresses, node) + } + } + return ingresses, nil +} + +// GetAllEgresses - gets all the hosts that are egresses +func GetAllEgresses() ([]models.Node, error) { + nodes, err := GetAllNodes() + if err != nil { + return nil, err + } + egresses := make([]models.Node, 0) + for _, node := range nodes { + if node.IsEgressGateway { + egresses = append(egresses, node) + } + } + return egresses, nil +} + // CreateEgressGateway - creates an egress gateway func CreateEgressGateway(gateway models.EgressGatewayRequest) (models.Node, error) { node, err := GetNodeByID(gateway.NodeID) @@ -28,10 +58,13 @@ func CreateEgressGateway(gateway models.EgressGatewayRequest) (models.Node, erro return models.Node{}, errors.New("firewall is not supported for egress gateways") } for i := len(gateway.Ranges) - 1; i >= 0; i-- { + // check if internet gateway IPv4 + if gateway.Ranges[i] == "0.0.0.0/0" && FreeTier { + return models.Node{}, fmt.Errorf("currently IPv4 internet gateways are not supported on the free tier: %s", gateway.Ranges[i]) + } + // check if internet gateway IPv6 if gateway.Ranges[i] == "::/0" { - logger.Log(0, "currently IPv6 internet gateways are not supported", gateway.Ranges[i]) - gateway.Ranges = append(gateway.Ranges[:i], gateway.Ranges[i+1:]...) - continue + return models.Node{}, fmt.Errorf("currently IPv6 internet gateways are not supported: %s", gateway.Ranges[i]) } normalized, err := NormalizeCIDR(gateway.Ranges[i]) if err != nil { @@ -150,15 +183,6 @@ func DeleteIngressGateway(nodeid string) (models.Node, bool, []models.ExtClient, node.IsIngressGateway = false node.IngressGatewayRange = "" node.Failover = false - - //logger.Log(3, "deleting ingress gateway firewall in use is '", host.FirewallInUse, "' and isEgressGateway is", node.IsEgressGateway) - if node.EgressGatewayRequest.NodeID != "" { - _, err := CreateEgressGateway(node.EgressGatewayRequest) - if err != nil { - logger.Log(0, fmt.Sprintf("failed to create egress gateway on node [%s] on network [%s]: %v", - node.EgressGatewayRequest.NodeID, node.EgressGatewayRequest.NetID, err)) - } - } err = UpsertNode(&node) if err != nil { return models.Node{}, wasFailover, removedClients, err diff --git a/logic/hosts.go b/logic/hosts.go index e53ed730b..895f571e5 100644 --- a/logic/hosts.go +++ b/logic/hosts.go @@ -158,14 +158,14 @@ func GetHost(hostid string) (*models.Host, error) { // CreateHost - creates a host if not exist func CreateHost(h *models.Host) error { - hosts, err := GetAllHosts() - if err != nil && !database.IsEmptyRecord(err) { - return err + hosts, hErr := GetAllHosts() + clients, cErr := GetAllExtClients() + if (hErr != nil && !database.IsEmptyRecord(hErr)) || + (cErr != nil && !database.IsEmptyRecord(cErr)) || + len(hosts)+len(clients) >= MachinesLimit { + return errors.New("free tier limits exceeded on machines") } - if len(hosts) >= Hosts_Limit { - return errors.New("free tier limits exceeded on hosts") - } - _, err = GetHost(h.ID.String()) + _, err := GetHost(h.ID.String()) if (err != nil && !database.IsEmptyRecord(err)) || (err == nil) { return ErrHostExists } diff --git a/logic/serverconf.go b/logic/serverconf.go index 4d73980dd..c307c29d5 100644 --- a/logic/serverconf.go +++ b/logic/serverconf.go @@ -2,22 +2,23 @@ package logic import ( "encoding/json" - "github.com/gravitl/netmaker/database" "github.com/gravitl/netmaker/servercfg" ) var ( - // Networks_Limit - dummy var for community - Networks_Limit = 1000000000 - // Users_Limit - dummy var for community - Users_Limit = 1000000000 - // Clients_Limit - dummy var for community - Clients_Limit = 1000000000 - // Hosts_Limit - dummy var for community - Hosts_Limit = 1000000000 - // Free_Tier - specifies if free tier - Free_Tier = false + // NetworksLimit - dummy var for community + NetworksLimit = 1000000000 + // UsersLimit - dummy var for community + UsersLimit = 1000000000 + // MachinesLimit - dummy var for community + MachinesLimit = 1000000000 + // IngressesLimit - dummy var for community + IngressesLimit = 1000000000 + // EgressesLimit - dummy var for community + EgressesLimit = 1000000000 + // FreeTier - specifies if free tier + FreeTier = false ) type serverData struct { @@ -87,10 +88,12 @@ func StoreJWTSecret(privateKey string) error { return database.Insert("nm-jwt-secret", string(data), database.SERVERCONF_TABLE_NAME) } +// SetFreeTierLimits - sets limits for free tier func SetFreeTierLimits() { - Free_Tier = true - Users_Limit = servercfg.GetUserLimit() - Clients_Limit = servercfg.GetClientLimit() - Networks_Limit = servercfg.GetNetworkLimit() - Hosts_Limit = servercfg.GetHostLimit() + FreeTier = true + UsersLimit = servercfg.GetUserLimit() + NetworksLimit = servercfg.GetNetworkLimit() + MachinesLimit = servercfg.GetMachinesLimit() + IngressesLimit = servercfg.GetIngressLimit() + EgressesLimit = servercfg.GetEgressLimit() } diff --git a/servercfg/serverconf.go b/servercfg/serverconf.go index 9767690e3..4a535c188 100644 --- a/servercfg/serverconf.go +++ b/servercfg/serverconf.go @@ -753,26 +753,28 @@ func GetNetworkLimit() int { return networkslimit } -// GetClientLimit - fetches free tier limits on ext. clients -func GetClientLimit() int { - var clientsLimit int - if os.Getenv("CLIENTS_LIMIT") != "" { - clientsLimit, _ = strconv.Atoi(os.Getenv("CLIENTS_LIMIT")) - } else { - clientsLimit = config.Config.Server.ClientsLimit +// GetMachinesLimit - fetches free tier limits on machines (clients + hosts) +func GetMachinesLimit() int { + if l, err := strconv.Atoi(os.Getenv("MACHINES_LIMIT")); err == nil { + return l } - return clientsLimit + return config.Config.Server.MachinesLimit } -// GetHostLimit - fetches free tier limits on hosts -func GetHostLimit() int { - var hostsLimit int - if os.Getenv("HOSTS_LIMIT") != "" { - hostsLimit, _ = strconv.Atoi(os.Getenv("HOSTS_LIMIT")) - } else { - hostsLimit = config.Config.Server.HostsLimit +// GetIngressLimit - fetches free tier limits on ingresses +func GetIngressLimit() int { + if l, err := strconv.Atoi(os.Getenv("INGRESSES_LIMIT")); err == nil { + return l + } + return config.Config.Server.IngressesLimit +} + +// GetEgressLimit - fetches free tier limits on egresses +func GetEgressLimit() int { + if l, err := strconv.Atoi(os.Getenv("EGRESSES_LIMIT")); err == nil { + return l } - return hostsLimit + return config.Config.Server.EgressesLimit } // DeployedByOperator - returns true if the instance is deployed by netmaker operator From b86bdc6c5524e920099d2ffecd14ce29b37679e5 Mon Sep 17 00:00:00 2001 From: Farukh Khan Date: Thu, 10 Aug 2023 22:57:44 +0800 Subject: [PATCH 25/33] NET-507 (#2506) * NET-507 * Fixed server restart changing the node expiration date to the defaults. * Removed expired nodes removal from zombie cleanup routine. * Added a new expired nodes deletion routine which removes expired nodes every hour. * NET-507 suggested changes * Possible fix for zombie nodes upon node deletion from the UI. * Suggested changes implemented for expired nodes deletion go routine. * NET-507 typo fix * typo fix for DeleteNode purge parameter --- controllers/hosts.go | 6 +++--- logic/nodes.go | 31 +++++++++++++++++++++++++++++++ logic/zombie.go | 4 ++-- main.go | 1 + models/node.go | 4 +++- 5 files changed, 40 insertions(+), 6 deletions(-) diff --git a/controllers/hosts.go b/controllers/hosts.go index f712c25d1..1cff4aea8 100644 --- a/controllers/hosts.go +++ b/controllers/hosts.go @@ -328,13 +328,13 @@ func deleteHostFromNetwork(w http.ResponseWriter, r *http.Request) { // unset all the relayed nodes logic.SetRelayedNodes(false, node.ID.String(), node.RelayedNodes) } - node.Action = models.NODE_DELETE - node.PendingDelete = true - logger.Log(1, "deleting node", node.ID.String(), "from host", currHost.Name) + logger.Log(1, "deleting node", node.ID.String(), "from host", currHost.Name) if err := logic.DeleteNode(node, forceDelete); err != nil { logic.ReturnErrorResponse(w, r, logic.FormatError(fmt.Errorf("failed to delete node"), "internal")) return } + node.Action = models.NODE_DELETE + node.PendingDelete = true // notify node change runUpdates(node, false) go func() { // notify of peer change diff --git a/logic/nodes.go b/logic/nodes.go index 93ed106dd..7c85b9982 100644 --- a/logic/nodes.go +++ b/logic/nodes.go @@ -1,6 +1,7 @@ package logic import ( + "context" "encoding/json" "errors" "fmt" @@ -20,6 +21,7 @@ import ( "github.com/gravitl/netmaker/models" "github.com/gravitl/netmaker/servercfg" "github.com/gravitl/netmaker/validation" + "golang.org/x/exp/slog" ) var ( @@ -444,6 +446,35 @@ func GetAllNodesAPI(nodes []models.Node) []models.ApiNode { return apiNodes[:] } +// DeleteExpiredNodes - goroutine which deletes nodes which are expired +func DeleteExpiredNodes(ctx context.Context, peerUpdate chan *models.Node) { + for { + select { + case <-ctx.Done(): + return + case <-time.After(time.Hour): + // Delete Expired Nodes Every Hour + allnodes, err := GetAllNodes() + if err != nil { + slog.Error("failed to retrieve all nodes", "error", err.Error()) + return + } + for _, node := range allnodes { + if time.Now().After(node.ExpirationDateTime) { + if err := DeleteNode(&node, false); err != nil { + slog.Error("error deleting expired node", "nodeid", node.ID.String(), "error", err.Error()) + continue + } + node.Action = models.NODE_DELETE + node.PendingDelete = true + peerUpdate <- &node + slog.Info("deleting expired node", "nodeid", node.ID.String()) + } + } + } + } +} + // == PRO == func updateProNodeACLS(node *models.Node) error { diff --git a/logic/zombie.go b/logic/zombie.go index e51365fc0..c7e858a88 100644 --- a/logic/zombie.go +++ b/logic/zombie.go @@ -37,7 +37,7 @@ func CheckZombies(newnode *models.Node) { //skip self continue } - if node.HostID == newnode.HostID || time.Now().After(node.ExpirationDateTime) { + if node.HostID == newnode.HostID { logger.Log(0, "adding ", node.ID.String(), " to zombie list") newZombie <- node.ID } @@ -97,7 +97,7 @@ func ManageZombies(ctx context.Context, peerUpdate chan *models.Node) { zombies = append(zombies[:i], zombies[i+1:]...) continue } - if time.Since(node.LastCheckIn) > time.Minute*ZOMBIE_DELETE_TIME || time.Now().After(node.ExpirationDateTime) { + if time.Since(node.LastCheckIn) > time.Minute*ZOMBIE_DELETE_TIME { if err := DeleteNode(&node, true); err != nil { logger.Log(1, "error deleting zombie node", zombies[i].String(), err.Error()) continue diff --git a/main.go b/main.go index 0bf365e5a..a0fe2824f 100644 --- a/main.go +++ b/main.go @@ -168,6 +168,7 @@ func runMessageQueue(wg *sync.WaitGroup, ctx context.Context) { go func() { peerUpdate := make(chan *models.Node) go logic.ManageZombies(ctx, peerUpdate) + go logic.DeleteExpiredNodes(ctx, peerUpdate) for nodeUpdate := range peerUpdate { if err := mq.NodeUpdate(nodeUpdate); err != nil { logger.Log(0, "failed to send peer update for deleted node: ", nodeUpdate.ID.String(), err.Error()) diff --git a/models/node.go b/models/node.go index a4104ac39..ee10f5597 100644 --- a/models/node.go +++ b/models/node.go @@ -341,7 +341,9 @@ func (node *Node) SetLastPeerUpdate() { // Node.SetExpirationDateTime - sets node expiry time func (node *Node) SetExpirationDateTime() { - node.ExpirationDateTime = time.Now().Add(TEN_YEARS_IN_SECONDS) + if node.ExpirationDateTime.IsZero() { + node.ExpirationDateTime = time.Now().Add(TEN_YEARS_IN_SECONDS) + } } // Node.SetDefaultName - sets a random name to node From 3e5f0e374e19a25fcc8df6635cc9ca6371521bd1 Mon Sep 17 00:00:00 2001 From: Farukh Khan Date: Sat, 12 Aug 2023 00:28:16 +0800 Subject: [PATCH 26/33] NET-517 (#2510) * Fixed emqx docker compose file syntax and added volumes for persistence. * Added helper comments for docker-compose.yml and netmaker.env files to enable emqx broker support. --- compose/docker-compose-emqx.yml | 19 ++++++++++++++----- compose/docker-compose.yml | 5 ++++- scripts/netmaker.default.env | 2 +- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/compose/docker-compose-emqx.yml b/compose/docker-compose-emqx.yml index 861e0693d..61dabf08e 100644 --- a/compose/docker-compose-emqx.yml +++ b/compose/docker-compose-emqx.yml @@ -4,14 +4,23 @@ services: mq: container_name: mq image: emqx/emqx:5.0.9 + env_file: ./netmaker.env restart: unless-stopped environment: - - EMQX_NAME: "emqx" + - EMQX_NAME=emqx - EMQX_DASHBOARD__DEFAULT_PASSWORD=${MQ_PASSWORD} - EMQX_DASHBOARD__DEFAULT_USERNAME=${MQ_USERNAME} ports: - - "1883:1883" # MQTT - - "8883:8883" # SSL MQTT - - "8083:8083" # Websockets + - "1883:1883" # MQTT + - "8883:8883" # SSL MQTT + - "8083:8083" # Websockets + - "8084:8084" # SSL Websockets - "18083:18083" # Dashboard/REST_API - + volumes: + - emqx_data:/opt/emqx/data + - emqx_etc:/opt/emqx/etc + - emqx_logs:/opt/emqx/log +volumes: + emqx_data: { } # storage for emqx data + emqx_etc: { } # storage for emqx etc + emqx_logs: { } # storage for emqx logs diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index e7b5db1f5..d17275291 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -14,7 +14,10 @@ services: # config-dependant vars - STUN_LIST=stun1.netmaker.io:3478,stun2.netmaker.io:3478,stun1.l.google.com:19302,stun2.l.google.com:19302 # The domain/host IP indicating the mq broker address - - BROKER_ENDPOINT=wss://broker.${NM_DOMAIN} + - BROKER_ENDPOINT=wss://broker.${NM_DOMAIN} # For EMQX broker use `BROKER_ENDPOINT=wss://broker.${NM_DOMAIN}/mqtt` + # For EMQX broker (uncomment the two lines below) + #- BROKER_TYPE=emqx + #- EMQX_REST_ENDPOINT=http://mq:18083 # The base domain of netmaker - SERVER_NAME=${NM_DOMAIN} - SERVER_API_CONN_STRING=api.${NM_DOMAIN}:443 diff --git a/scripts/netmaker.default.env b/scripts/netmaker.default.env index 799df19f7..e714c19a1 100644 --- a/scripts/netmaker.default.env +++ b/scripts/netmaker.default.env @@ -41,7 +41,7 @@ DISPLAY_KEYS="on" DATABASE="sqlite" # The address of the mq server. If running from docker compose it will be "mq". Otherwise, need to input address. # If using "host networking", it will find and detect the IP of the mq container. -SERVER_BROKER_ENDPOINT="ws://mq:1883" +SERVER_BROKER_ENDPOINT="ws://mq:1883" # For EMQX websockets use `SERVER_BROKER_ENDPOINT=ws://mq:8083/mqtt` # The reachable port of STUN on the server STUN_PORT="3478" # Logging verbosity level - 1, 2, or 3 From 333a8e59b71eaaf1d25ce906a7e46a2ff046102b Mon Sep 17 00:00:00 2001 From: Matthew R Kasun Date: Mon, 14 Aug 2023 13:44:10 -0400 Subject: [PATCH 27/33] migration (#2509) * create gateways during migration * set version for testing * restruct migration * debug logging * enforce unique names for ext client names (#2476) * enforce unique names for ext client names * only check for unique id on creation * check for unique id if changed * prune(NET-483): remove defunct host.internetgateway field (#2487) * don't reference host on err (#2493) * deprecrate netclient install scripts (#2490) * Net 500: validate network parameter passed to node endpoints (#2480) * enforce unique names for ext client names * only check for unique id on creation * check for unique id if changed * validate network parameter passed to node endpoints --------- Co-authored-by: Abhishek K <32607604+abhishek9686@users.noreply.github.com> * NET-513 (#2492) nm-certs.sh now requests certificate for EE and CE edition domains accordingly. * [NET-404] Run in limited mode when ee checks fail (#2474) * Add limited http handlers functionality to rest handler * Export ee.errValidation (ee.ErrValidation) * Export a fatal error handled by the hook manager * Export a new status variable for unlicensed server * Mark server as unlicensed when ee checks fail * Handle license validation failures with a (re)boot in a limited state * Revert "Export a fatal error handled by the hook manager" This reverts commit 069c21974a8d36e889c73ad78023448d787d62a5. * Revert "Export ee.errValidation (ee.ErrValidation)" This reverts commit 59dbab8c79773ca5d879f28cbaf53f3dd4297b9b. * Revert "Add limited http handlers functionality to rest handler" This reverts commit e2f1f28facaca54713db76a588839cd2733cf673. * Revert "Handle license validation failures with a (re)boot in a limited state" This reverts commit 58cfbbaf522a1345aac1fa67964ebff0a6d60cd8. * Revert "Mark server as unlicensed when ee checks fail" This reverts commit 77c6dbdd3c9cfa6e7d6becedef6251e8617ae367. * Handle license validation failures with a middleware * Forbid responses if unlicensed ee and not in status api * Remove unused func * feat(NET-449): add sync feature to request a host pull from server (#2491) * fix(NET-486): change client name length validation (#2498) set limit to 5<=x<=32 * [NET-477] Pick AMB URL dynamically (#2489) * Introduce config for environment * Introduce func to get environment * Choose accounts api host from environment * Test the ee package on workflows * Use build tag ee for license_test.go * [Feature]: nm-quick script tackling arm TODO support (#2488) * domain flag for auto installs * use static servers with custom domain (#2421) * send delete peer update always * fix add/remove host api calls * keep mq updates in a single go func * move branch test logic to devops (#2443) * handle IOT OS * save server name to env (#2460) * ensure branch test servers available after test runs (#2467) * save server name to env * free server always; add PR to discord messages * use correct method to delete droplets (#2468) * quick fix for the launcher * removed exit when triggering not supported exit and removed the TODO comments related to this issue --------- Co-authored-by: Matthew R Kasun Co-authored-by: Alex Feiszli <31018251+afeiszli@users.noreply.github.com> Co-authored-by: Christopher Blaha Co-authored-by: Abhishek Kondur Co-authored-by: Abhishek K <32607604+abhishek9686@users.noreply.github.com> * rebase conflict * include pass and os in mirgration data * node network ranges * remove debugging logs * add gateways * use sent node * upgrade shell script * associate node to host during migration * add node to host.Nodes and publish peer update * save host outside loop * fix script name * simplify upgrade script * don't migrate relays * simplify upgrade script even more * guard against blank address or address6 * typos --------- Co-authored-by: Aceix Co-authored-by: Abhishek K <32607604+abhishek9686@users.noreply.github.com> Co-authored-by: Farukh Khan Co-authored-by: Gabriel de Souza Seibel Co-authored-by: bornav <51048565+bornav@users.noreply.github.com> Co-authored-by: Alex Feiszli <31018251+afeiszli@users.noreply.github.com> Co-authored-by: Christopher Blaha Co-authored-by: Abhishek Kondur --- controllers/migrate.go | 212 ++++++++++++--- models/migrate.go | 4 +- models/node.go | 6 +- scripts/nm-upgrade.sh | 574 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 747 insertions(+), 49 deletions(-) create mode 100755 scripts/nm-upgrade.sh diff --git a/controllers/migrate.go b/controllers/migrate.go index badc12734..fa03fccf9 100644 --- a/controllers/migrate.go +++ b/controllers/migrate.go @@ -2,15 +2,22 @@ package controller import ( "encoding/json" + "fmt" + "net" "net/http" + "strconv" + "time" - "github.com/gravitl/netmaker/auth" + "github.com/google/uuid" "github.com/gravitl/netmaker/database" "github.com/gravitl/netmaker/logger" "github.com/gravitl/netmaker/logic" "github.com/gravitl/netmaker/models" + "github.com/gravitl/netmaker/mq" "github.com/gravitl/netmaker/servercfg" "golang.org/x/crypto/bcrypt" + "golang.org/x/exp/slog" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" ) // swagger:route PUT /api/v1/nodes/migrate nodes migrateNode @@ -26,63 +33,182 @@ import ( // 200: nodeJoinResponse func migrate(w http.ResponseWriter, r *http.Request) { data := models.MigrationData{} + host := models.Host{} + node := models.Node{} + nodes := []models.Node{} + server := models.ServerConfig{} err := json.NewDecoder(r.Body).Decode(&data) if err != nil { logger.Log(0, r.Header.Get("user"), "error decoding request body: ", err.Error()) logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest")) return } - - var networksToAdd = []string{} - for i := range data.LegacyNodes { - legacyNode := data.LegacyNodes[i] - record, err := database.FetchRecord(database.NODES_TABLE_NAME, legacyNode.ID) + for i, legacy := range data.LegacyNodes { + record, err := database.FetchRecord(database.NODES_TABLE_NAME, legacy.ID) if err != nil { - logger.Log(0, "no record for legacy node", legacyNode.ID, err.Error()) - continue - } else { - var oldLegacyNode models.LegacyNode - if err = json.Unmarshal([]byte(record), &oldLegacyNode); err != nil { - logger.Log(0, "error decoding legacy node", err.Error()) + slog.Error("legacy node not found", "error", err) + logic.ReturnErrorResponse(w, r, logic.FormatError(fmt.Errorf("legacy node not found %w", err), "badrequest")) + return + } + var legacyNode models.LegacyNode + if err = json.Unmarshal([]byte(record), &legacyNode); err != nil { + slog.Error("decoding legacy node", "errror", err) + logic.ReturnErrorResponse(w, r, logic.FormatError(fmt.Errorf("decode legacy node %w", err), "badrequest")) + return + } + if err := bcrypt.CompareHashAndPassword([]byte(legacyNode.Password), []byte(legacy.Password)); err != nil { + slog.Error("legacy node invalid password", "error", err) + logic.ReturnErrorResponse(w, r, logic.FormatError(fmt.Errorf("invalid password %w", err), "unauthorized")) + return + } + if i == 0 { + host, node = convertLegacyHostNode(legacy) + host.Name = data.HostName + host.HostPass = data.Password + host.OS = data.OS + if err := logic.CreateHost(&host); err != nil { + slog.Error("create host", "error", err) logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest")) - continue + return + } + server = servercfg.GetServerInfo() + if servercfg.GetBrokerType() == servercfg.EmqxBrokerType { + server.MQUserName = host.ID.String() } - if err := bcrypt.CompareHashAndPassword([]byte(oldLegacyNode.Password), []byte(legacyNode.Password)); err != nil { - logger.Log(0, "error decoding legacy password", err.Error()) - logic.ReturnErrorResponse(w, r, logic.FormatError(err, "unauthorized")) - continue + key, keyErr := logic.RetrievePublicTrafficKey() + if keyErr != nil { + slog.Error("retrieving traffickey", "error", err) + logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal")) + return } - networksToAdd = append(networksToAdd, oldLegacyNode.Network) - _ = database.DeleteRecord(database.NODES_TABLE_NAME, oldLegacyNode.ID) + server.TrafficKey = key + } else { + node = convertLegacyNode(legacyNode, host.ID) } - } - if len(networksToAdd) == 0 { - logger.Log(0, "no valid networks to migrate for host", data.NewHost.Name) - logic.ReturnErrorResponse(w, r, logic.FormatError(err, "unauthorized")) - return - } - if !logic.HostExists(&data.NewHost) { - logic.CheckHostPorts(&data.NewHost) - if err = logic.CreateHost(&data.NewHost); err != nil { - logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest")) - return + if err := logic.UpsertNode(&node); err != nil { + slog.Error("update node", "error", err) + continue } + host.Nodes = append(host.Nodes, node.ID.String()) + + nodes = append(nodes, node) } - key, keyErr := logic.RetrievePublicTrafficKey() - if keyErr != nil { - logger.Log(0, "error retrieving key:", keyErr.Error()) - logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal")) - return + if err := logic.UpsertHost(&host); err != nil { + slog.Error("save host", "error", err) } - server := servercfg.GetServerInfo() - server.TrafficKey = key - response := models.RegisterResponse{ - ServerConf: server, - RequestedHost: data.NewHost, + go mq.PublishPeerUpdate() + response := models.HostPull{ + Host: host, + Nodes: nodes, + ServerConfig: server, } w.WriteHeader(http.StatusOK) json.NewEncoder(w).Encode(&response) - logger.Log(0, "successfully migrated host", data.NewHost.Name, data.NewHost.ID.String()) - // notify host of changes, peer and node updates - go auth.CheckNetRegAndHostUpdate(networksToAdd, &data.NewHost) + + slog.Info("migrated nodes") + // check for gateways + for _, node := range data.LegacyNodes { + if node.IsEgressGateway == "yes" { + egressGateway := models.EgressGatewayRequest{ + NodeID: node.ID, + Ranges: node.EgressGatewayRanges, + NatEnabled: node.EgressGatewayNatEnabled, + } + if _, err := logic.CreateEgressGateway(egressGateway); err != nil { + logger.Log(0, "error creating egress gateway for node", node.ID, err.Error()) + } + } + if node.IsIngressGateway == "yes" { + ingressGateway := models.IngressRequest{} + ingressNode, err := logic.CreateIngressGateway(node.Network, node.ID, ingressGateway) + if err != nil { + logger.Log(0, "error creating ingress gateway for node", node.ID, err.Error()) + } + runUpdates(&ingressNode, true) + } + } +} + +func convertLegacyHostNode(legacy models.LegacyNode) (models.Host, models.Node) { + //convert host + host := models.Host{} + host.ID = uuid.New() + host.IPForwarding = models.ParseBool(legacy.IPForwarding) + host.AutoUpdate = servercfg.AutoUpdateEnabled() + host.Interface = "netmaker" + host.ListenPort = int(legacy.ListenPort) + host.MTU = int(legacy.MTU) + host.PublicKey, _ = wgtypes.ParseKey(legacy.PublicKey) + host.MacAddress = net.HardwareAddr(legacy.MacAddress) + host.TrafficKeyPublic = legacy.TrafficKeys.Mine + host.Nodes = append([]string{}, legacy.ID) + host.Interfaces = legacy.Interfaces + //host.DefaultInterface = legacy.Defaul + host.EndpointIP = net.ParseIP(legacy.Endpoint) + host.IsDocker = models.ParseBool(legacy.IsDocker) + host.IsK8S = models.ParseBool(legacy.IsK8S) + host.IsStatic = models.ParseBool(legacy.IsStatic) + node := convertLegacyNode(legacy, host.ID) + return host, node +} + +func convertLegacyNode(legacy models.LegacyNode, hostID uuid.UUID) models.Node { + //convert node + node := models.Node{} + node.ID, _ = uuid.Parse(legacy.ID) + node.HostID = hostID + node.Network = legacy.Network + valid4 := true + valid6 := true + _, cidr4, err := net.ParseCIDR(legacy.NetworkSettings.AddressRange) + if err != nil { + valid4 = false + slog.Warn("parsing address range", "error", err) + } else { + node.NetworkRange = *cidr4 + } + _, cidr6, err := net.ParseCIDR(legacy.NetworkSettings.AddressRange6) + if err != nil { + valid6 = false + slog.Warn("parsing address range6", "error", err) + } else { + node.NetworkRange6 = *cidr6 + } + node.Server = servercfg.GetServer() + node.Connected = models.ParseBool(legacy.Connected) + if valid4 { + node.Address = net.IPNet{ + IP: net.ParseIP(legacy.Address), + Mask: cidr4.Mask, + } + } + if valid6 { + node.Address6 = net.IPNet{ + IP: net.ParseIP(legacy.Address6), + Mask: cidr6.Mask, + } + } + node.Action = models.NODE_NOOP + node.LocalAddress = net.IPNet{ + IP: net.ParseIP(legacy.LocalAddress), + } + node.IsEgressGateway = models.ParseBool(legacy.IsEgressGateway) + node.EgressGatewayRanges = legacy.EgressGatewayRanges + node.IsIngressGateway = models.ParseBool(legacy.IsIngressGateway) + node.IsRelayed = false + node.IsRelay = false + node.RelayedNodes = []string{} + node.DNSOn = models.ParseBool(legacy.DNSOn) + node.PersistentKeepalive = time.Duration(legacy.PersistentKeepalive) + node.LastModified = time.Now() + node.ExpirationDateTime, _ = time.Parse(strconv.Itoa(int(legacy.ExpirationDateTime)), "0") + node.EgressGatewayNatEnabled = models.ParseBool(legacy.EgressGatewayNatEnabled) + node.EgressGatewayRequest = legacy.EgressGatewayRequest + node.IngressGatewayRange = legacy.IngressGatewayRange + node.IngressGatewayRange6 = legacy.IngressGatewayRange6 + node.DefaultACL = legacy.DefaultACL + node.OwnerID = legacy.OwnerID + node.FailoverNode, _ = uuid.Parse(legacy.FailoverNode) + node.Failover = models.ParseBool(legacy.Failover) + return node } diff --git a/models/migrate.go b/models/migrate.go index 9972c22c1..978354cb4 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -2,6 +2,8 @@ package models // MigrationData struct needed to create new v0.18.0 node from v.0.17.X node type MigrationData struct { - NewHost Host + HostName string + Password string + OS string LegacyNodes []LegacyNode } diff --git a/models/node.go b/models/node.go index ee10f5597..ed778d00e 100644 --- a/models/node.go +++ b/models/node.go @@ -100,7 +100,6 @@ type Node struct { // LegacyNode - legacy struct for node model type LegacyNode struct { ID string `json:"id,omitempty" bson:"id,omitempty" yaml:"id,omitempty" validate:"required,min=5,id_unique"` - HostID string `json:"hostid,omitempty" bson:"id,omitempty" yaml:"hostid,omitempty" validate:"required,min=5,id_unique"` Address string `json:"address" bson:"address" yaml:"address" validate:"omitempty,ipv4"` Address6 string `json:"address6" bson:"address6" yaml:"address6" validate:"omitempty,ipv6"` LocalAddress string `json:"localaddress" bson:"localaddress" yaml:"localaddress" validate:"omitempty"` @@ -109,7 +108,6 @@ type LegacyNode struct { NetworkSettings Network `json:"networksettings" bson:"networksettings" yaml:"networksettings" validate:"-"` ListenPort int32 `json:"listenport" bson:"listenport" yaml:"listenport" validate:"omitempty,numeric,min=1024,max=65535"` LocalListenPort int32 `json:"locallistenport" bson:"locallistenport" yaml:"locallistenport" validate:"numeric,min=0,max=65535"` - ProxyListenPort int32 `json:"proxy_listen_port" bson:"proxy_listen_port" yaml:"proxy_listen_port" validate:"numeric,min=0,max=65535"` PublicKey string `json:"publickey" bson:"publickey" yaml:"publickey" validate:"required,base64"` Endpoint string `json:"endpoint" bson:"endpoint" yaml:"endpoint" validate:"required,ip"` AllowedIPs []string `json:"allowedips" bson:"allowedips" yaml:"allowedips"` @@ -153,8 +151,6 @@ type LegacyNode struct { FirewallInUse string `json:"firewallinuse" bson:"firewallinuse" yaml:"firewallinuse"` InternetGateway string `json:"internetgateway" bson:"internetgateway" yaml:"internetgateway"` Connected string `json:"connected" bson:"connected" yaml:"connected" validate:"checkyesorno"` - PendingDelete bool `json:"pendingdelete" bson:"pendingdelete" yaml:"pendingdelete"` - Proxy bool `json:"proxy" bson:"proxy" yaml:"proxy"` // == PRO == DefaultACL string `json:"defaultacl,omitempty" bson:"defaultacl,omitempty" yaml:"defaultacl,omitempty" validate:"checkyesornoorunset"` OwnerID string `json:"ownerid,omitempty" bson:"ownerid,omitempty" yaml:"ownerid,omitempty"` @@ -527,7 +523,7 @@ func (ln *LegacyNode) ConvertToNewNode() (*Host, *Node) { func (n *Node) Legacy(h *Host, s *ServerConfig, net *Network) *LegacyNode { l := LegacyNode{} l.ID = n.ID.String() - l.HostID = h.ID.String() + //l.HostID = h.ID.String() l.Address = n.Address.String() l.Address6 = n.Address6.String() l.Interfaces = h.Interfaces diff --git a/scripts/nm-upgrade.sh b/scripts/nm-upgrade.sh new file mode 100755 index 000000000..7da1ce6e9 --- /dev/null +++ b/scripts/nm-upgrade.sh @@ -0,0 +1,574 @@ +#!/bin/bash + +CONFIG_FILE=netmaker.env +# location of nm-quick.sh (usually `/root`) +SCRIPT_DIR=$(dirname "$(realpath "$0")") +CONFIG_PATH="$SCRIPT_DIR/$CONFIG_FILE" +NM_QUICK_VERSION="0.1.0" +LATEST=$(curl -s https://api.github.com/repos/gravitl/netmaker/releases/latest | grep "tag_name" | cut -d : -f 2,3 | tr -d [:space:],\") + +if [ "$(id -u)" -ne 0 ]; then + echo "This script must be run as root" + exit 1 +fi + +unset INSTALL_TYPE +unset NETMAKER_BASE_DOMAIN + +# usage - displays usage instructions +usage() { + echo "nm-upgrade.sh v$NM_QUICK_VERSION" + echo "usage: ./nm-upgrade.sh" + exit 1 +} + +while getopts v flag; do + case "${flag}" in + v) + usage + exit 0 + ;; + *) + usage + exit 0 + ;; + esac +done + +# print_logo - prints the netmaker logo +print_logo() { + cat <<"EOF" +- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + __ __ ______ ______ __ __ ______ __ __ ______ ______ +/\ "-.\ \ /\ ___\ /\__ _\ /\ "-./ \ /\ __ \ /\ \/ / /\ ___\ /\ == \ +\ \ \-. \ \ \ __\ \/_/\ \/ \ \ \-./\ \ \ \ __ \ \ \ _"-. \ \ __\ \ \ __< + \ \_\\"\_\ \ \_____\ \ \_\ \ \_\ \ \_\ \ \_\ \_\ \ \_\ \_\ \ \_____\ \ \_\ \_\ + \/_/ \/_/ \/_____/ \/_/ \/_/ \/_/ \/_/\/_/ \/_/\/_/ \/_____/ \/_/ /_/ + + +- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +EOF +} + +# set_buildinfo - sets the information based on script input for how the installation should be run +set_buildinfo() { + + MASTERKEY=$(grep MASTER_KEY docker-compose.yml | awk '{print $2;}' | tr -d '"') + EMAIL=$(grep email Caddyfile | awk '{print $2;}' | tr -d '"') + BROKER=$(grep SERVER_NAME docker-compose.yml | awk '{print $2;}' | tr -d '"') + PREFIX="broker." + NETMAKER_BASE_DOMAIN=${BROKER/#$PREFIX} + + + echo "-----------------------------------------------------" + echo "Would you like to install Netmaker Community Edition (CE), or Netmaker Enterprise Edition (EE)?" + echo "EE will require you to create an account at https://app.netmaker.io" + echo "-----------------------------------------------------" + select install_option in "Community Edition" "Enterprise Edition"; do + case $REPLY in + 1) + echo "installing Netmaker CE" + INSTALL_TYPE="ce" + break + ;; + 2) + echo "installing Netmaker EE" + INSTALL_TYPE="ee" + break + ;; + *) echo "invalid option $REPLY" ;; + esac + done + + echo "-----------Build Options-----------------------------" + echo " EE or CE: $INSTALL_TYPE" + echo " Version: $LATEST" + echo " Installer: v$NM_QUICK_VERSION" + echo "-----------------------------------------------------" + +} + +# install_yq - install yq if not present +install_yq() { + if ! command -v yq &>/dev/null; then + wget -qO /usr/bin/yq https://github.com/mikefarah/yq/releases/download/v4.31.1/yq_linux_$(dpkg --print-architecture) + chmod +x /usr/bin/yq + fi + set +e + if ! command -v yq &>/dev/null; then + set -e + wget -qO /usr/bin/yq https://github.com/mikefarah/yq/releases/download/v4.31.1/yq_linux_amd64 + chmod +x /usr/bin/yq + fi + set -e + if ! command -v yq &>/dev/null; then + echo "failed to install yq. Please install yq and try again." + echo "https://github.com/mikefarah/yq/#install" + exit 1 + fi +} + +# install and run upgrade tool +upgrade() { + wget -qO /tmp/nm-upgrade https://fileserver.netmaker.io/upgrade/nm-upgrade-${ARCH} + chmod +x /tmp/nm-upgrade + echo "generating netclient configuration files" + /tmp/nm-upgrade +} + +# setup_netclient - installs netclient +setup_netclient() { + wget -qO netclient https://github.com/gravitl/netclient/releases/download/$LATEST/netclient-linux-$ARCH + chmod +x netclient + ./netclient install -v 3 +} + + +# wait_seconds - wait for the specified period of time +wait_seconds() { ( + for ((a = 1; a <= $1; a++)); do + echo ". . ." + sleep 1 + done +); } + +# confirm - get user input to confirm that they want to perform the next step +confirm() { ( + while true; do + read -p 'Does everything look right? [y/n]: ' yn + case $yn in + [Yy]*) + override="true" + break + ;; + [Nn]*) + echo "exiting..." + exit 1 + # TODO start from the beginning instead + ;; + *) echo "Please answer yes or no." ;; + esac + done +) } + +save_config() { ( + echo "Saving the config to $CONFIG_PATH" + touch "$CONFIG_PATH" + save_config_item NM_EMAIL "$EMAIL" + save_config_item NM_DOMAIN "$NETMAKER_BASE_DOMAIN" + save_config_item UI_IMAGE_TAG "$LATEST" + # version-specific entries + if [ "$INSTALL_TYPE" = "ee" ]; then + save_config_item NETMAKER_TENANT_ID "$TENANT_ID" + save_config_item LICENSE_KEY "$LICENSE_KEY" + save_config_item METRICS_EXPORTER "on" + save_config_item PROMETHEUS "on" + ave_config_item SERVER_IMAGE_TAG "$LATEST-ee" + else + save_config_item METRICS_EXPORTER "off" + save_config_item PROMETHEUS "off" + save_config_item SERVER_IMAGE_TAG "$LATEST" + fi + # copy entries from the previous config + local toCopy=("SERVER_HOST" "MASTER_KEY" "TURN_USERNAME" "TURN_PASSWORD" "MQ_USERNAME" "MQ_PASSWORD" + "INSTALL_TYPE" "NODE_ID" "DNS_MODE" "NETCLIENT_AUTO_UPDATE" "API_PORT" + "CORS_ALLOWED_ORIGIN" "DISPLAY_KEYS" "DATABASE" "SERVER_BROKER_ENDPOINT" "STUN_PORT" "VERBOSITY" + "TURN_PORT" "USE_TURN" "DEBUG_MODE" "TURN_API_PORT" "REST_BACKEND" + "DISABLE_REMOTE_IP_CHECK" "NETCLIENT_ENDPOINT_DETECTION" "TELEMETRY" "AUTH_PROVIDER" "CLIENT_ID" "CLIENT_SECRET" + "FRONTEND_URL" "AZURE_TENANT" "OIDC_ISSUER" "EXPORTER_API_PORT") + for name in "${toCopy[@]}"; do + save_config_item $name "${!name}" + done + # preserve debug entries + if test -n "$NM_SKIP_BUILD"; then + save_config_item NM_SKIP_BUILD "$NM_SKIP_BUILD" + fi + if test -n "$NM_SKIP_CLONE"; then + save_config_item NM_SKIP_CLONE "$NM_SKIP_CLONE" + fi + if test -n "$NM_SKIP_DEPS"; then + save_config_item NM_SKIP_DEPS "$NM_SKIP_DEPS" + fi +); } + +save_config_item() { ( + local NAME="$1" + local VALUE="$2" + #echo "$NAME=$VALUE" + if test -z "$VALUE"; then + # load the default for empty values + VALUE=$(awk -F'=' "/^$NAME/ { print \$2}" "$SCRIPT_DIR/netmaker.default.env") + # trim quotes for docker + VALUE=$(echo "$VALUE" | sed -E "s|^(['\"])(.*)\1$|\2|g") + #echo "Default for $NAME=$VALUE" + fi + # TODO single quote passwords + if grep -q "^$NAME=" "$CONFIG_PATH"; then + # TODO escape | in the value + sed -i "s|$NAME=.*|$NAME=$VALUE|" "$CONFIG_PATH" + else + echo "$NAME=$VALUE" >>"$CONFIG_PATH" + fi +); } + +# install_dependencies - install necessary packages to run netmaker +install_dependencies() { + + if test -n "$NM_SKIP_DEPS"; then + return + fi + + echo "checking dependencies..." + + OS=$(uname) + if [ -f /etc/debian_version ]; then + dependencies="git wireguard wireguard-tools dnsutils jq docker.io docker-compose grep gawk" + update_cmd='apt update' + install_cmd='apt-get install -y' + elif [ -f /etc/alpine-release ]; then + dependencies="git wireguard jq docker.io docker-compose grep gawk" + update_cmd='apk update' + install_cmd='apk --update add' + elif [ -f /etc/centos-release ]; then + dependencies="git wireguard jq bind-utils docker.io docker-compose grep gawk" + update_cmd='yum update' + install_cmd='yum install -y' + elif [ -f /etc/fedora-release ]; then + dependencies="git wireguard bind-utils jq docker.io docker-compose grep gawk" + update_cmd='dnf update' + install_cmd='dnf install -y' + elif [ -f /etc/redhat-release ]; then + dependencies="git wireguard jq docker.io bind-utils docker-compose grep gawk" + update_cmd='yum update' + install_cmd='yum install -y' + elif [ -f /etc/arch-release ]; then + dependencies="git wireguard-tools dnsutils jq docker.io docker-compose grep gawk" + update_cmd='pacman -Sy' + install_cmd='pacman -S --noconfirm' + elif [ "${OS}" = "FreeBSD" ]; then + dependencies="git wireguard wget jq docker.io docker-compose grep gawk" + update_cmd='pkg update' + install_cmd='pkg install -y' + else + install_cmd='' + fi + + if [ -z "${install_cmd}" ]; then + echo "OS unsupported for automatic dependency install" + # TODO shouldnt exit, check if deps available, if not + # ask the user to install manually and continue when ready + exit 1 + fi + # TODO add other supported architectures + ARCH=$(uname -m) + if [ "$ARCH" = "x86_64" ]; then + ARCH=amd64 + elif [ "$ARCH" = "aarch64" ]; then + ARCH=arm64 + else + echo "Unsupported architechure" + # exit 1 + fi + set -- $dependencies + + ${update_cmd} + + while [ -n "$1" ]; do + if [ "${OS}" = "FreeBSD" ]; then + is_installed=$(pkg check -d $1 | grep "Checking" | grep "done") + if [ "$is_installed" != "" ]; then + echo " " $1 is installed + else + echo " " $1 is not installed. Attempting install. + ${install_cmd} $1 + sleep 5 + is_installed=$(pkg check -d $1 | grep "Checking" | grep "done") + if [ "$is_installed" != "" ]; then + echo " " $1 is installed + elif [ -x "$(command -v $1)" ]; then + echo " " $1 is installed + else + echo " " FAILED TO INSTALL $1 + echo " " This may break functionality. + fi + fi + else + if [ "${OS}" = "OpenWRT" ] || [ "${OS}" = "TurrisOS" ]; then + is_installed=$(opkg list-installed $1 | grep $1) + else + is_installed=$(dpkg-query -W --showformat='${Status}\n' $1 | grep "install ok installed") + fi + if [ "${is_installed}" != "" ]; then + echo " " $1 is installed + else + echo " " $1 is not installed. Attempting install. + ${install_cmd} $1 + sleep 5 + if [ "${OS}" = "OpenWRT" ] || [ "${OS}" = "TurrisOS" ]; then + is_installed=$(opkg list-installed $1 | grep $1) + else + is_installed=$(dpkg-query -W --showformat='${Status}\n' $1 | grep "install ok installed") + fi + if [ "${is_installed}" != "" ]; then + echo " " $1 is installed + elif [ -x "$(command -v $1)" ]; then + echo " " $1 is installed + else + echo " " FAILED TO INSTALL $1 + echo " " This may break functionality. + fi + fi + fi + shift + done + + echo "-----------------------------------------------------" + echo "dependency check complete" + echo "-----------------------------------------------------" +} + +# set_install_vars - sets the variables that will be used throughout installation +set_install_vars() { + + IP_ADDR=$(dig -4 myip.opendns.com @resolver1.opendns.com +short) + if [ "$IP_ADDR" = "" ]; then + IP_ADDR=$(curl -s ifconfig.me) + fi + if [ "$NETMAKER_BASE_DOMAIN" = "" ]; then + NETMAKER_BASE_DOMAIN=nm.$(echo $IP_ADDR | tr . -).nip.io + fi + SERVER_HOST=$IP_ADDR + if test -z "$MASTER_KEY"; then + MASTER_KEY=$( + tr -dc A-Za-z0-9 &1) + + if [[ "$i" == 8 ]]; then + echo " Caddy is having an issue setting up certificates, please investigate (docker logs caddy)" + echo " Exiting..." + exit 1 + elif [[ "$curlresponse" == *"failed to verify the legitimacy of the server"* ]]; then + echo " Certificates not yet configured, retrying..." + + elif [[ "$curlresponse" == *"left intact"* ]]; then + echo " Certificates ok" + break + else + secs=$(($i * 5 + 10)) + echo " Issue establishing connection...retrying in $secs seconds..." + fi + sleep $secs + done + +} + +# print_success - prints a success message upon completion +print_success() { + echo "-----------------------------------------------------------------" + echo "-----------------------------------------------------------------" + echo "Netmaker setup is now complete. You are ready to begin using Netmaker." + echo "Visit dashboard.$NETMAKER_BASE_DOMAIN to log in" + echo "-----------------------------------------------------------------" + echo "-----------------------------------------------------------------" +} + +cleanup() { + echo "Stopping all containers..." + local containers=("mq" "netmaker-ui" "coredns" "turn" "caddy" "netmaker" "netmaker-exporter" "prometheus" "grafana") + for name in "${containers[@]}"; do + local running=$(docker ps | grep -w "$name") + local exists=$(docker ps -a | grep -w "$name") + if test -n "$running"; then + docker stop "$name" 1>/dev/null + fi + if test -n "$exists"; then + docker rm "$name" 1>/dev/null + fi + done +} + +# print netmaker logo +print_logo + +# read the config +if [ -f "$CONFIG_PATH" ]; then + echo "Using config: $CONFIG_PATH" + source "$CONFIG_PATH" + if [ "$UPGRADE_FLAG" = "yes" ]; then + INSTALL_TYPE="ee" + fi +fi + +# setup the build instructions +set_buildinfo + +set +e + +# install necessary packages +install_dependencies + +# install yq if necessary +install_yq + +set -e + +# get user input for variables +set_install_vars + +set +e +cleanup +set -e + +# get upgrade tool and run +upgrade + +# get and set config files, startup docker-compose +install_netmaker + +set +e + +# make sure Caddy certs are working +test_connection + +set -e + +# install netclient +setup_netclient + + +# print success message +print_success + From b8b3e2f9df9be8508477c3f8ba07a78d57287d65 Mon Sep 17 00:00:00 2001 From: Matthew R Kasun Date: Mon, 14 Aug 2023 15:23:03 -0400 Subject: [PATCH 28/33] dns tests --- controllers/dns_test.go | 6 ++++-- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/controllers/dns_test.go b/controllers/dns_test.go index e070cfed4..5c2453b05 100644 --- a/controllers/dns_test.go +++ b/controllers/dns_test.go @@ -202,13 +202,15 @@ func TestCreateDNS(t *testing.T) { func TestSetDNS(t *testing.T) { deleteAllDNS(t) deleteAllNetworks() + etc, err := os.Stat("/etc/hosts") + assert.Nil(t, err) t.Run("NoNetworks", func(t *testing.T) { err := logic.SetDNS() assert.Nil(t, err) info, err := os.Stat("./config/dnsconfig/netmaker.hosts") assert.Nil(t, err) assert.False(t, info.IsDir()) - assert.Equal(t, int64(0), info.Size()) + assert.Equal(t, etc.Size(), info.Size()) }) t.Run("NoEntries", func(t *testing.T) { createNet() @@ -217,7 +219,7 @@ func TestSetDNS(t *testing.T) { info, err := os.Stat("./config/dnsconfig/netmaker.hosts") assert.Nil(t, err) assert.False(t, info.IsDir()) - assert.Equal(t, int64(0), info.Size()) + assert.Equal(t, etc.Size(), info.Size()) }) t.Run("NodeExists", func(t *testing.T) { createTestNode() diff --git a/go.mod b/go.mod index f73dba38c..36dae93c3 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/rqlite/gorqlite v0.0.0-20210514125552-08ff1e76b22f github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/stretchr/testify v1.8.4 - github.com/txn2/txeh v1.4.0 + github.com/txn2/txeh v1.5.3 golang.org/x/crypto v0.12.0 golang.org/x/net v0.14.0 // indirect golang.org/x/oauth2 v0.11.0 diff --git a/go.sum b/go.sum index 537e31453..75ae359df 100644 --- a/go.sum +++ b/go.sum @@ -103,8 +103,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/txn2/txeh v1.4.0 h1:0tdvpA4HGJrj8X3kmrU6o/JFStI009nKxwDpMK5CnRU= -github.com/txn2/txeh v1.4.0/go.mod h1:Mgq0hY184zCrDBLgvkIp+9NYGHoYbJcu4xKqUcx1shc= +github.com/txn2/txeh v1.5.3 h1:ZMgc3r+5/AFtE/ayCoICpvxj7xl/CYsZjnIGhozV/Kc= +github.com/txn2/txeh v1.5.3/go.mod h1:qYzGG9kCzeVEI12geK4IlanHWY8X4uy/I3NcW7mk8g4= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c h1:3lbZUMbMiGUW/LMkfsEABsc5zNT9+b1CvsJx47JzJ8g= github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c/go.mod h1:UrdRz5enIKZ63MEE3IF9l2/ebyx59GyGgPi+tICQdmM= From 64cea1dff5943f6ca777c120df9c2f542c9cfd6b Mon Sep 17 00:00:00 2001 From: Matthew R Kasun Date: Mon, 14 Aug 2023 15:26:55 -0400 Subject: [PATCH 29/33] go mod tidy --- go.sum | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.sum b/go.sum index 537e31453..75ae359df 100644 --- a/go.sum +++ b/go.sum @@ -103,8 +103,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/txn2/txeh v1.4.0 h1:0tdvpA4HGJrj8X3kmrU6o/JFStI009nKxwDpMK5CnRU= -github.com/txn2/txeh v1.4.0/go.mod h1:Mgq0hY184zCrDBLgvkIp+9NYGHoYbJcu4xKqUcx1shc= +github.com/txn2/txeh v1.5.3 h1:ZMgc3r+5/AFtE/ayCoICpvxj7xl/CYsZjnIGhozV/Kc= +github.com/txn2/txeh v1.5.3/go.mod h1:qYzGG9kCzeVEI12geK4IlanHWY8X4uy/I3NcW7mk8g4= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c h1:3lbZUMbMiGUW/LMkfsEABsc5zNT9+b1CvsJx47JzJ8g= github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c/go.mod h1:UrdRz5enIKZ63MEE3IF9l2/ebyx59GyGgPi+tICQdmM= From 13c379f7512ae3009d4f4ce80d717aa846985521 Mon Sep 17 00:00:00 2001 From: Matthew R Kasun Date: Tue, 15 Aug 2023 16:40:29 -0400 Subject: [PATCH 30/33] set dns dir when testing --- controllers/dns_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/controllers/dns_test.go b/controllers/dns_test.go index 5c2453b05..e1f2cfee3 100644 --- a/controllers/dns_test.go +++ b/controllers/dns_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + "github.com/gravitl/netmaker/functions" "github.com/gravitl/netmaker/logic" "github.com/gravitl/netmaker/models" ) @@ -50,6 +51,8 @@ func TestGetNodeDNS(t *testing.T) { deleteAllNetworks() createNet() createHost() + err := functions.SetDNSDir() + assert.Nil(t, err) t.Run("NoNodes", func(t *testing.T) { dns, _ := logic.GetNodeDNS("skynet") assert.Equal(t, []models.DNSEntry(nil), dns) @@ -204,6 +207,8 @@ func TestSetDNS(t *testing.T) { deleteAllNetworks() etc, err := os.Stat("/etc/hosts") assert.Nil(t, err) + err = functions.SetDNSDir() + assert.Nil(t, err) t.Run("NoNetworks", func(t *testing.T) { err := logic.SetDNS() assert.Nil(t, err) From a7d25f8ad7505cf314b6c9daa465a7d80fc431d2 Mon Sep 17 00:00:00 2001 From: Matthew R Kasun Date: Tue, 15 Aug 2023 18:24:10 -0400 Subject: [PATCH 31/33] troubleshoot test failure --- controllers/dns_test.go | 5 +++++ go.mod | 5 ++++- go.sum | 44 +++++++++-------------------------------- 3 files changed, 18 insertions(+), 36 deletions(-) diff --git a/controllers/dns_test.go b/controllers/dns_test.go index e1f2cfee3..4d242d409 100644 --- a/controllers/dns_test.go +++ b/controllers/dns_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + "github.com/gravitl/netclient/ncutils" "github.com/gravitl/netmaker/functions" "github.com/gravitl/netmaker/logic" "github.com/gravitl/netmaker/models" @@ -207,6 +208,8 @@ func TestSetDNS(t *testing.T) { deleteAllNetworks() etc, err := os.Stat("/etc/hosts") assert.Nil(t, err) + out, _ := ncutils.RunCmd("cat /etc/hosts", false) + t.Log(out) err = functions.SetDNSDir() assert.Nil(t, err) t.Run("NoNetworks", func(t *testing.T) { @@ -216,6 +219,8 @@ func TestSetDNS(t *testing.T) { assert.Nil(t, err) assert.False(t, info.IsDir()) assert.Equal(t, etc.Size(), info.Size()) + out, _ := ncutils.RunCmd("cat ./config/dnsconfig/netmaker.hosts", false) + t.Log(out) }) t.Run("NoEntries", func(t *testing.T) { createNet() diff --git a/go.mod b/go.mod index 36dae93c3..4638134be 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( golang.org/x/oauth2 v0.11.0 golang.org/x/sys v0.11.0 // indirect golang.org/x/text v0.12.0 // indirect - golang.zx2c4.com/wireguard/wgctrl v0.0.0-20220324164955-056925b7df31 + golang.zx2c4.com/wireguard/wgctrl v0.0.0-20221104135756-97bc4ad4a1cb google.golang.org/protobuf v1.31.0 // indirect gopkg.in/yaml.v3 v3.0.1 ) @@ -40,6 +40,7 @@ require ( require ( github.com/devilcove/httpclient v0.6.0 github.com/go-jose/go-jose/v3 v3.0.0 + github.com/gravitl/netclient v0.20.5 github.com/guumaster/tablewriter v0.0.10 github.com/matryer/is v1.4.1 github.com/olekukonko/tablewriter v0.0.5 @@ -52,6 +53,8 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/vishvananda/netlink v1.1.0 // indirect + github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df // indirect ) require ( diff --git a/go.sum b/go.sum index 75ae359df..c83455493 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,6 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -51,13 +49,14 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gravitl/netclient v0.20.5 h1:NfSCV6jd8YDuZ8qjBH6HJeS8AFVoxjHoMe3uSRw5nVY= +github.com/gravitl/netclient v0.20.5/go.mod h1:sU8Zg3+Rd8vRiD5EfxTTbhinyo/S3GcXhN2cSb390wk= github.com/guumaster/tablewriter v0.0.10 h1:A0HD94yMdt4usgxBjoEceNeE0XMJ027euoHAzsPqBQs= github.com/guumaster/tablewriter v0.0.10/go.mod h1:p4FRFhyfo0UD9ZLmMRbbJooTUsxo6b80qZTERVDWrH8= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= @@ -70,10 +69,6 @@ github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/mdlayher/genetlink v1.2.0/go.mod h1:ra5LDov2KrUCZJiAtEvXXZBxGMInICMXIwshlJ+qRxQ= -github.com/mdlayher/netlink v1.6.0/go.mod h1:0o3PlBmGst1xve7wQ7j/hwpNaFaH4qCRyWCdcZk8/vA= -github.com/mdlayher/socket v0.1.1/go.mod h1:mYV5YIZAfHh4dzDVzI8x8tWLWCliuX8Mon5Awbj+qDs= -github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721/go.mod h1:Ickgr2WtCLZ2MDGd4Gr0geeCH5HybhRJbonOgQpvSxc= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -106,60 +101,39 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/txn2/txeh v1.5.3 h1:ZMgc3r+5/AFtE/ayCoICpvxj7xl/CYsZjnIGhozV/Kc= github.com/txn2/txeh v1.5.3/go.mod h1:qYzGG9kCzeVEI12geK4IlanHWY8X4uy/I3NcW7mk8g4= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= +github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c h1:3lbZUMbMiGUW/LMkfsEABsc5zNT9+b1CvsJx47JzJ8g= github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c/go.mod h1:UrdRz5enIKZ63MEE3IF9l2/ebyx59GyGgPi+tICQdmM= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220208050332-20e1d8d225ab/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211111083644-e5c967477495/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211110154304-99a53858aa08/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220207234003-57398862261d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.zx2c4.com/go118/netip v0.0.0-20211111135330-a4a02eeacf9d/go.mod h1:5yyfuiqVIJ7t+3MqrpTQ+QqRkMWiESiyDvPNvKYCecg= -golang.zx2c4.com/wintun v0.0.0-20211104114900-415007cec224/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI= -golang.zx2c4.com/wireguard v0.0.0-20220202223031-3b95c81cc178/go.mod h1:TjUWrnD5ATh7bFvmm/ALEJZQ4ivKbETb6pmyj1vUoNI= -golang.zx2c4.com/wireguard/wgctrl v0.0.0-20220324164955-056925b7df31 h1:AgW3hljgTzuRbCB0j+q9tXT0uy6ij7vMjEzSCeMlQY0= -golang.zx2c4.com/wireguard/wgctrl v0.0.0-20220324164955-056925b7df31/go.mod h1:8P32Ilp1kCpwB4ItaHyvSk4xAtnpQ+8gQVfg5WaO1TU= +golang.zx2c4.com/wireguard/wgctrl v0.0.0-20221104135756-97bc4ad4a1cb h1:9aqVcYEDHmSNb0uOWukxV5lHV09WqiSiCuhEgWNETLY= +golang.zx2c4.com/wireguard/wgctrl v0.0.0-20221104135756-97bc4ad4a1cb/go.mod h1:mQqgjkW8GQQcJQsbBvK890TKqUK1DfKWkuBGbOkuMHQ= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= From 4445752bf7b2305f7cbe16738651cafa911a4a9b Mon Sep 17 00:00:00 2001 From: Matthew R Kasun Date: Tue, 15 Aug 2023 18:32:36 -0400 Subject: [PATCH 32/33] test fail troubleshooting --- controllers/dns_test.go | 19 +++++++------------ go.mod | 3 --- go.sum | 7 ------- 3 files changed, 7 insertions(+), 22 deletions(-) diff --git a/controllers/dns_test.go b/controllers/dns_test.go index 4d242d409..125c5a603 100644 --- a/controllers/dns_test.go +++ b/controllers/dns_test.go @@ -9,7 +9,6 @@ import ( "github.com/stretchr/testify/assert" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" - "github.com/gravitl/netclient/ncutils" "github.com/gravitl/netmaker/functions" "github.com/gravitl/netmaker/logic" "github.com/gravitl/netmaker/models" @@ -206,30 +205,26 @@ func TestCreateDNS(t *testing.T) { func TestSetDNS(t *testing.T) { deleteAllDNS(t) deleteAllNetworks() - etc, err := os.Stat("/etc/hosts") + etc, err := os.ReadFile("/etc/hosts") assert.Nil(t, err) - out, _ := ncutils.RunCmd("cat /etc/hosts", false) - t.Log(out) err = functions.SetDNSDir() assert.Nil(t, err) t.Run("NoNetworks", func(t *testing.T) { err := logic.SetDNS() assert.Nil(t, err) - info, err := os.Stat("./config/dnsconfig/netmaker.hosts") + info, err := os.ReadFile("./config/dnsconfig/netmaker.hosts") assert.Nil(t, err) - assert.False(t, info.IsDir()) - assert.Equal(t, etc.Size(), info.Size()) - out, _ := ncutils.RunCmd("cat ./config/dnsconfig/netmaker.hosts", false) - t.Log(out) + //assert.False(t, info.IsDir()) + assert.Equal(t, etc, info) + t.Log(string(etc), string(info)) }) t.Run("NoEntries", func(t *testing.T) { createNet() err := logic.SetDNS() assert.Nil(t, err) - info, err := os.Stat("./config/dnsconfig/netmaker.hosts") + info, err := os.ReadFile("./config/dnsconfig/netmaker.hosts") assert.Nil(t, err) - assert.False(t, info.IsDir()) - assert.Equal(t, etc.Size(), info.Size()) + assert.Equal(t, etc, info) }) t.Run("NodeExists", func(t *testing.T) { createTestNode() diff --git a/go.mod b/go.mod index 4638134be..4152bb28d 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,6 @@ require ( require ( github.com/devilcove/httpclient v0.6.0 github.com/go-jose/go-jose/v3 v3.0.0 - github.com/gravitl/netclient v0.20.5 github.com/guumaster/tablewriter v0.0.10 github.com/matryer/is v1.4.1 github.com/olekukonko/tablewriter v0.0.5 @@ -53,8 +52,6 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/vishvananda/netlink v1.1.0 // indirect - github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df // indirect ) require ( diff --git a/go.sum b/go.sum index c83455493..35a122031 100644 --- a/go.sum +++ b/go.sum @@ -49,8 +49,6 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gravitl/netclient v0.20.5 h1:NfSCV6jd8YDuZ8qjBH6HJeS8AFVoxjHoMe3uSRw5nVY= -github.com/gravitl/netclient v0.20.5/go.mod h1:sU8Zg3+Rd8vRiD5EfxTTbhinyo/S3GcXhN2cSb390wk= github.com/guumaster/tablewriter v0.0.10 h1:A0HD94yMdt4usgxBjoEceNeE0XMJ027euoHAzsPqBQs= github.com/guumaster/tablewriter v0.0.10/go.mod h1:p4FRFhyfo0UD9ZLmMRbbJooTUsxo6b80qZTERVDWrH8= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= @@ -101,10 +99,6 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/txn2/txeh v1.5.3 h1:ZMgc3r+5/AFtE/ayCoICpvxj7xl/CYsZjnIGhozV/Kc= github.com/txn2/txeh v1.5.3/go.mod h1:qYzGG9kCzeVEI12geK4IlanHWY8X4uy/I3NcW7mk8g4= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= -github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c h1:3lbZUMbMiGUW/LMkfsEABsc5zNT9+b1CvsJx47JzJ8g= github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c/go.mod h1:UrdRz5enIKZ63MEE3IF9l2/ebyx59GyGgPi+tICQdmM= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -123,7 +117,6 @@ golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From 31d4c3eb79fe3b409675f4e8cb89015e98a0389b Mon Sep 17 00:00:00 2001 From: Matthew R Kasun Date: Tue, 15 Aug 2023 19:06:19 -0400 Subject: [PATCH 33/33] use txeh.RenderHostFile to compare --- controllers/dns_test.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/controllers/dns_test.go b/controllers/dns_test.go index 125c5a603..37563987a 100644 --- a/controllers/dns_test.go +++ b/controllers/dns_test.go @@ -7,6 +7,7 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/assert" + "github.com/txn2/txeh" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" "github.com/gravitl/netmaker/functions" @@ -205,26 +206,28 @@ func TestCreateDNS(t *testing.T) { func TestSetDNS(t *testing.T) { deleteAllDNS(t) deleteAllNetworks() - etc, err := os.ReadFile("/etc/hosts") + etc, err := txeh.NewHosts(&txeh.HostsConfig{}) assert.Nil(t, err) err = functions.SetDNSDir() assert.Nil(t, err) t.Run("NoNetworks", func(t *testing.T) { err := logic.SetDNS() assert.Nil(t, err) - info, err := os.ReadFile("./config/dnsconfig/netmaker.hosts") + info, err := txeh.NewHosts(&txeh.HostsConfig{ + ReadFilePath: "./config/dnsconfig/netmaker.hosts", + }) assert.Nil(t, err) - //assert.False(t, info.IsDir()) - assert.Equal(t, etc, info) - t.Log(string(etc), string(info)) + assert.Equal(t, etc.RenderHostsFile(), info.RenderHostsFile()) }) t.Run("NoEntries", func(t *testing.T) { createNet() err := logic.SetDNS() assert.Nil(t, err) - info, err := os.ReadFile("./config/dnsconfig/netmaker.hosts") + info, err := txeh.NewHosts(&txeh.HostsConfig{ + ReadFilePath: "./config/dnsconfig/netmaker.hosts", + }) assert.Nil(t, err) - assert.Equal(t, etc, info) + assert.Equal(t, etc.RenderHostsFile(), info.RenderHostsFile()) }) t.Run("NodeExists", func(t *testing.T) { createTestNode()