From fb29deaa8329a103f3767232e4abc5367d59c9a3 Mon Sep 17 00:00:00 2001 From: gabrielseibel1 Date: Fri, 28 Jul 2023 09:04:04 -0300 Subject: [PATCH 01/14] Add limited http handlers functionality to rest handler --- controllers/controller.go | 17 +++++++++++------ ee/initialize.go | 4 ++-- main.go | 2 +- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/controllers/controller.go b/controllers/controller.go index 7abef20b3..c969e2d36 100644 --- a/controllers/controller.go +++ b/controllers/controller.go @@ -3,19 +3,19 @@ package controller import ( "context" "fmt" + "github.com/gorilla/handlers" "net/http" "strings" "sync" "time" - "github.com/gorilla/handlers" "github.com/gorilla/mux" "github.com/gravitl/netmaker/logger" "github.com/gravitl/netmaker/servercfg" ) -// HttpHandlers - handler functions for REST interactions -var HttpHandlers = []interface{}{ +// FullHttpHandlers - handler functions for REST interactions +var FullHttpHandlers = []func(r *mux.Router){ nodeHandlers, userHandlers, networkHandlers, @@ -30,8 +30,13 @@ var HttpHandlers = []interface{}{ legacyHandlers, } +// LimitedHttpHandlers - limited handler functions for REST interactions +var LimitedHttpHandlers = []func(r *mux.Router){ + serverHandlers, +} + // HandleRESTRequests - handles the rest requests -func HandleRESTRequests(wg *sync.WaitGroup, ctx context.Context) { +func HandleRESTRequests(wg *sync.WaitGroup, ctx context.Context, httpHandlers []func(r *mux.Router)) { defer wg.Done() r := mux.NewRouter() @@ -42,8 +47,8 @@ func HandleRESTRequests(wg *sync.WaitGroup, ctx context.Context) { originsOk := handlers.AllowedOrigins(strings.Split(servercfg.GetAllowedOrigin(), ",")) methodsOk := handlers.AllowedMethods([]string{http.MethodGet, http.MethodPut, http.MethodPost, http.MethodDelete}) - for _, handler := range HttpHandlers { - handler.(func(*mux.Router))(r) + for _, handler := range httpHandlers { + handler(r) } port := servercfg.GetAPIPort() diff --git a/ee/initialize.go b/ee/initialize.go index 455ee59df..afd6b8096 100644 --- a/ee/initialize.go +++ b/ee/initialize.go @@ -18,8 +18,8 @@ func InitEE() { setIsEnterprise() servercfg.Is_EE = true models.SetLogo(retrieveEELogo()) - controller.HttpHandlers = append( - controller.HttpHandlers, + controller.FullHttpHandlers = append( + controller.FullHttpHandlers, ee_controllers.MetricHandlers, ee_controllers.NetworkUsersHandlers, ee_controllers.UserGroupsHandlers, diff --git a/main.go b/main.go index 6abd97680..cecdb4217 100644 --- a/main.go +++ b/main.go @@ -136,7 +136,7 @@ func startControllers(wg *sync.WaitGroup, ctx context.Context) { } } wg.Add(1) - go controller.HandleRESTRequests(wg, ctx) + go controller.HandleRESTRequests(wg, ctx, controller.FullHttpHandlers) } //Run MessageQueue if servercfg.IsMessageQueueBackend() { From 89dacc046c49efdc697cf3a62333375032d727a7 Mon Sep 17 00:00:00 2001 From: gabrielseibel1 Date: Fri, 28 Jul 2023 09:06:31 -0300 Subject: [PATCH 02/14] Export ee.errValidation (ee.ErrValidation) --- ee/types.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ee/types.go b/ee/types.go index 17b5dd331..f4dc92444 100644 --- a/ee/types.go +++ b/ee/types.go @@ -1,6 +1,8 @@ package ee -import "fmt" +import ( + "errors" +) const ( api_endpoint = "https://api.accounts.netmaker.io/api/v1/license/validate" @@ -9,7 +11,7 @@ const ( server_id_key = "nm-server-id" ) -var errValidation = fmt.Errorf(license_validation_err_msg) +var ErrValidation = errors.New(license_validation_err_msg) // LicenseKey - the license key struct representation with associated data type LicenseKey struct { From 3c6f6f9da2aac78c93093d28a894e7958ac115fd Mon Sep 17 00:00:00 2001 From: gabrielseibel1 Date: Fri, 28 Jul 2023 09:07:11 -0300 Subject: [PATCH 03/14] Export a fatal error handled by the hook manager --- logic/timer.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/logic/timer.go b/logic/timer.go index 89070f2fe..a6ab0aac1 100644 --- a/logic/timer.go +++ b/logic/timer.go @@ -2,7 +2,10 @@ package logic import ( "context" + "errors" "fmt" + "golang.org/x/exp/slog" + "os" "sync" "time" @@ -18,6 +21,9 @@ const timer_hours_between_runs = 24 // HookManagerCh - channel to add any new hooks var HookManagerCh = make(chan models.HookDetails, 2) +// HookManagerFatalError is an error to which is fatal to the hook manager (stops the program!) +var HookManagerFatalError = errors.New("fatal periodic procedure") + // == Public == // TimerCheckpoint - Checks if 24 hours has passed since telemetry was last sent. If so, sends telemetry data to posthog @@ -70,7 +76,13 @@ func addHookWithInterval(ctx context.Context, wg *sync.WaitGroup, hook func() er case <-ctx.Done(): return case <-ticker.C: - hook() + if err := hook(); err != nil { + // stop the server if there was a fatal error + if errors.Is(err, HookManagerFatalError) { + slog.Error(err.Error()) + os.Exit(0) + } + } } } From 9e520b69a04280b9e5b36f3da41ff35ee0ce9aba Mon Sep 17 00:00:00 2001 From: gabrielseibel1 Date: Fri, 28 Jul 2023 09:08:21 -0300 Subject: [PATCH 04/14] Export a new status variable for unlicensed server --- controllers/server.go | 18 ++++++------------ servercfg/serverconf.go | 5 +++-- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/controllers/server.go b/controllers/server.go index e9aa1b760..ba4a18f67 100644 --- a/controllers/server.go +++ b/controllers/server.go @@ -68,22 +68,16 @@ func getUsage(w http.ResponseWriter, r *http.Request) { // Responses: // 200: serverConfigResponse func getStatus(w http.ResponseWriter, r *http.Request) { - // TODO - // - check health of broker type status struct { - DB bool `json:"db_connected"` - Broker bool `json:"broker_connected"` - Usage struct { - Hosts int `json:"hosts"` - Clients int `json:"clients"` - Networks int `json:"networks"` - Users int `json:"users"` - } `json:"usage"` + DB bool `json:"db_connected"` + Broker bool `json:"broker_connected"` + UnlicensedEE bool `json:"unlicensed_ee"` } currentServerStatus := status{ - DB: database.IsConnected(), - Broker: mq.IsConnected(), + DB: database.IsConnected(), + Broker: mq.IsConnected(), + UnlicensedEE: servercfg.Is_EE && servercfg.IsUnlicensed, } w.Header().Set("Content-Type", "application/json") diff --git a/servercfg/serverconf.go b/servercfg/serverconf.go index 18170a7eb..101ba94f5 100644 --- a/servercfg/serverconf.go +++ b/servercfg/serverconf.go @@ -18,8 +18,9 @@ import ( const EmqxBrokerType = "emqx" var ( - Version = "dev" - Is_EE = false + Version = "dev" + Is_EE = false + IsUnlicensed = false ) // SetHost - sets the host ip From aee1e80a803ff149362cf2ab3e23dba3159fb6bb Mon Sep 17 00:00:00 2001 From: gabrielseibel1 Date: Fri, 28 Jul 2023 09:10:08 -0300 Subject: [PATCH 05/14] Mark server as unlicensed when ee checks fail --- ee/initialize.go | 9 +++++++-- logic/server.go | 9 ++++++--- main.go | 6 +++++- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/ee/initialize.go b/ee/initialize.go index afd6b8096..ad659af50 100644 --- a/ee/initialize.go +++ b/ee/initialize.go @@ -11,6 +11,7 @@ import ( "github.com/gravitl/netmaker/logic" "github.com/gravitl/netmaker/models" "github.com/gravitl/netmaker/servercfg" + "golang.org/x/exp/slog" ) // InitEE - Initialize EE Logic @@ -25,14 +26,18 @@ func InitEE() { ee_controllers.UserGroupsHandlers, ee_controllers.RelayHandlers, ) - logic.EnterpriseCheckFuncs = append(logic.EnterpriseCheckFuncs, func() { + logic.EnterpriseCheckFuncs = append(logic.EnterpriseCheckFuncs, func() error { // == License Handling == - ValidateLicense() + if err := ValidateLicense(); err != nil { + slog.Error(err.Error()) + return err + } logger.Log(0, "proceeding with Paid Tier license") logic.SetFreeTierForTelemetry(false) // == End License Handling == AddLicenseHooks() resetFailover() + return nil }) logic.EnterpriseFailoverFunc = eelogic.SetFailover logic.EnterpriseResetFailoverFunc = eelogic.ResetFailover diff --git a/logic/server.go b/logic/server.go index a8d509326..39bd1cb25 100644 --- a/logic/server.go +++ b/logic/server.go @@ -6,7 +6,7 @@ import ( ) // EnterpriseCheckFuncs - can be set to run functions for EE -var EnterpriseCheckFuncs []func() +var EnterpriseCheckFuncs []func() error // EnterpriseFailoverFunc - interface to control failover funcs var EnterpriseFailoverFunc func(node *models.Node) error @@ -26,8 +26,11 @@ const KUBERNETES_LISTEN_PORT = 31821 const KUBERNETES_SERVER_MTU = 1024 // EnterpriseCheck - Runs enterprise functions if presented -func EnterpriseCheck() { +func EnterpriseCheck() error { for _, check := range EnterpriseCheckFuncs { - check() + if err := check(); err != nil { + return err + } } + return nil } diff --git a/main.go b/main.go index cecdb4217..de2647a6a 100644 --- a/main.go +++ b/main.go @@ -91,7 +91,11 @@ func initialize() { // Client Mode Prereq Check if err != nil { logger.Log(1, "Timer error occurred: ", err.Error()) } - logic.EnterpriseCheck() + + if err := logic.EnterpriseCheck(); err != nil { + servercfg.IsUnlicensed = true + return + } var authProvider = auth.InitializeAuthProvider() if authProvider != "" { From a4febe031214e9eb69051c0c2711d0716816026d Mon Sep 17 00:00:00 2001 From: gabrielseibel1 Date: Fri, 28 Jul 2023 09:11:10 -0300 Subject: [PATCH 06/14] Handle license validation failures with a (re)boot in a limited state --- ee/license.go | 38 ++++++++++++++++++++++---------------- main.go | 8 ++++++++ 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/ee/license.go b/ee/license.go index dfbb36eca..385062b25 100644 --- a/ee/license.go +++ b/ee/license.go @@ -12,7 +12,6 @@ import ( "golang.org/x/exp/slog" "io" "net/http" - "os" "time" "github.com/gravitl/netmaker/database" @@ -35,7 +34,14 @@ type apiServerConf struct { // AddLicenseHooks - adds the validation and cache clear hooks func AddLicenseHooks() { logic.HookManagerCh <- models.HookDetails{ - Hook: ValidateLicense, + Hook: func() error { + if err := ValidateLicense(); err != nil { + // stop the program when license is not valid anymore + // if the server restarts and still fails the license check, it can reboot in a limited mode + return fmt.Errorf("%w: %s", logic.HookManagerFatalError, err.Error()) + } + return nil + }, Interval: time.Hour, } logic.HookManagerCh <- models.HookDetails{ @@ -48,25 +54,26 @@ func AddLicenseHooks() { // checks if a license is valid + limits are not exceeded // if license is free_tier and limits exceeds, then server should terminate // if license is not valid, server should terminate +// TODO update comment func ValidateLicense() error { licenseKeyValue := servercfg.GetLicenseKey() netmakerTenantID := servercfg.GetNetmakerTenantID() slog.Info("proceeding with Netmaker license validation...") if len(licenseKeyValue) == 0 { - failValidation(errors.New("empty license-key (LICENSE_KEY environment variable)")) + return wrappedInErrValidation(errors.New("empty license-key (LICENSE_KEY environment variable)")) } if len(netmakerTenantID) == 0 { - failValidation(errors.New("empty tenant-id (NETMAKER_TENANT_ID environment variable)")) + return wrappedInErrValidation(errors.New("empty tenant-id (NETMAKER_TENANT_ID environment variable)")) } apiPublicKey, err := getLicensePublicKey(licenseKeyValue) if err != nil { - failValidation(fmt.Errorf("failed to get license public key: %w", err)) + return wrappedInErrValidation(fmt.Errorf("failed to get license public key: %w", err)) } tempPubKey, tempPrivKey, err := FetchApiServerKeys() if err != nil { - failValidation(fmt.Errorf("failed to fetch api server keys: %w", err)) + return wrappedInErrValidation(fmt.Errorf("failed to fetch api server keys: %w", err)) } licenseSecret := LicenseSecret{ @@ -76,35 +83,35 @@ func ValidateLicense() error { secretData, err := json.Marshal(&licenseSecret) if err != nil { - failValidation(fmt.Errorf("failed to marshal license secret: %w", err)) + return wrappedInErrValidation(fmt.Errorf("failed to marshal license secret: %w", err)) } encryptedData, err := ncutils.BoxEncrypt(secretData, apiPublicKey, tempPrivKey) if err != nil { - failValidation(fmt.Errorf("failed to encrypt license secret data: %w", err)) + return wrappedInErrValidation(fmt.Errorf("failed to encrypt license secret data: %w", err)) } validationResponse, err := validateLicenseKey(encryptedData, tempPubKey) if err != nil { - failValidation(fmt.Errorf("failed to validate license key: %w", err)) + return wrappedInErrValidation(fmt.Errorf("failed to validate license key: %w", err)) } if len(validationResponse) == 0 { - failValidation(errors.New("empty validation response")) + return wrappedInErrValidation(errors.New("empty validation response")) } var licenseResponse ValidatedLicense if err = json.Unmarshal(validationResponse, &licenseResponse); err != nil { - failValidation(fmt.Errorf("failed to unmarshal validation response: %w", err)) + return wrappedInErrValidation(fmt.Errorf("failed to unmarshal validation response: %w", err)) } respData, err := ncutils.BoxDecrypt(base64decode(licenseResponse.EncryptedLicense), apiPublicKey, tempPrivKey) if err != nil { - failValidation(fmt.Errorf("failed to decrypt license: %w", err)) + return wrappedInErrValidation(fmt.Errorf("failed to decrypt license: %w", err)) } license := LicenseKey{} if err = json.Unmarshal(respData, &license); err != nil { - failValidation(fmt.Errorf("failed to unmarshal license key: %w", err)) + return wrappedInErrValidation(fmt.Errorf("failed to unmarshal license key: %w", err)) } slog.Info("License validation succeeded!") @@ -158,9 +165,8 @@ func FetchApiServerKeys() (pub *[32]byte, priv *[32]byte, err error) { return pub, priv, nil } -func failValidation(err error) { - slog.Error(errValidation.Error(), "error", err) - os.Exit(0) +func wrappedInErrValidation(err error) error { + return fmt.Errorf("%w: %s", ErrValidation, err.Error()) } func getLicensePublicKey(licensePubKeyEncoded string) (*[32]byte, error) { diff --git a/main.go b/main.go index de2647a6a..3449fd012 100644 --- a/main.go +++ b/main.go @@ -124,6 +124,14 @@ func initialize() { // Client Mode Prereq Check } func startControllers(wg *sync.WaitGroup, ctx context.Context) { + // limit the controllers when unlicensed + if servercfg.IsUnlicensed { + wg.Add(2) + go controller.HandleRESTRequests(wg, ctx, controller.LimitedHttpHandlers) + go logic.StartHookManager(ctx, wg) + return + } + if servercfg.IsDNSMode() { err := logic.SetDNS() if err != nil { From 199dc0612323b61568f7e2d33c0b4d47161d90a1 Mon Sep 17 00:00:00 2001 From: gabrielseibel1 Date: Mon, 31 Jul 2023 15:08:58 -0300 Subject: [PATCH 07/14] Revert "Export a fatal error handled by the hook manager" This reverts commit 069c21974a8d36e889c73ad78023448d787d62a5. --- logic/timer.go | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/logic/timer.go b/logic/timer.go index a6ab0aac1..89070f2fe 100644 --- a/logic/timer.go +++ b/logic/timer.go @@ -2,10 +2,7 @@ package logic import ( "context" - "errors" "fmt" - "golang.org/x/exp/slog" - "os" "sync" "time" @@ -21,9 +18,6 @@ const timer_hours_between_runs = 24 // HookManagerCh - channel to add any new hooks var HookManagerCh = make(chan models.HookDetails, 2) -// HookManagerFatalError is an error to which is fatal to the hook manager (stops the program!) -var HookManagerFatalError = errors.New("fatal periodic procedure") - // == Public == // TimerCheckpoint - Checks if 24 hours has passed since telemetry was last sent. If so, sends telemetry data to posthog @@ -76,13 +70,7 @@ func addHookWithInterval(ctx context.Context, wg *sync.WaitGroup, hook func() er case <-ctx.Done(): return case <-ticker.C: - if err := hook(); err != nil { - // stop the server if there was a fatal error - if errors.Is(err, HookManagerFatalError) { - slog.Error(err.Error()) - os.Exit(0) - } - } + hook() } } From 2d181543b5d4bc9a63476b81ce9b311388793250 Mon Sep 17 00:00:00 2001 From: gabrielseibel1 Date: Mon, 31 Jul 2023 15:09:11 -0300 Subject: [PATCH 08/14] Revert "Export ee.errValidation (ee.ErrValidation)" This reverts commit 59dbab8c79773ca5d879f28cbaf53f3dd4297b9b. --- ee/types.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ee/types.go b/ee/types.go index f4dc92444..17b5dd331 100644 --- a/ee/types.go +++ b/ee/types.go @@ -1,8 +1,6 @@ package ee -import ( - "errors" -) +import "fmt" const ( api_endpoint = "https://api.accounts.netmaker.io/api/v1/license/validate" @@ -11,7 +9,7 @@ const ( server_id_key = "nm-server-id" ) -var ErrValidation = errors.New(license_validation_err_msg) +var errValidation = fmt.Errorf(license_validation_err_msg) // LicenseKey - the license key struct representation with associated data type LicenseKey struct { From 7973944b8745b7150904dd0f43c954d6fd1e05fd Mon Sep 17 00:00:00 2001 From: gabrielseibel1 Date: Mon, 31 Jul 2023 15:09:23 -0300 Subject: [PATCH 09/14] Revert "Add limited http handlers functionality to rest handler" This reverts commit e2f1f28facaca54713db76a588839cd2733cf673. --- controllers/controller.go | 17 ++++++----------- ee/initialize.go | 4 ++-- main.go | 2 +- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/controllers/controller.go b/controllers/controller.go index c969e2d36..7abef20b3 100644 --- a/controllers/controller.go +++ b/controllers/controller.go @@ -3,19 +3,19 @@ package controller import ( "context" "fmt" - "github.com/gorilla/handlers" "net/http" "strings" "sync" "time" + "github.com/gorilla/handlers" "github.com/gorilla/mux" "github.com/gravitl/netmaker/logger" "github.com/gravitl/netmaker/servercfg" ) -// FullHttpHandlers - handler functions for REST interactions -var FullHttpHandlers = []func(r *mux.Router){ +// HttpHandlers - handler functions for REST interactions +var HttpHandlers = []interface{}{ nodeHandlers, userHandlers, networkHandlers, @@ -30,13 +30,8 @@ var FullHttpHandlers = []func(r *mux.Router){ legacyHandlers, } -// LimitedHttpHandlers - limited handler functions for REST interactions -var LimitedHttpHandlers = []func(r *mux.Router){ - serverHandlers, -} - // HandleRESTRequests - handles the rest requests -func HandleRESTRequests(wg *sync.WaitGroup, ctx context.Context, httpHandlers []func(r *mux.Router)) { +func HandleRESTRequests(wg *sync.WaitGroup, ctx context.Context) { defer wg.Done() r := mux.NewRouter() @@ -47,8 +42,8 @@ func HandleRESTRequests(wg *sync.WaitGroup, ctx context.Context, httpHandlers [] originsOk := handlers.AllowedOrigins(strings.Split(servercfg.GetAllowedOrigin(), ",")) methodsOk := handlers.AllowedMethods([]string{http.MethodGet, http.MethodPut, http.MethodPost, http.MethodDelete}) - for _, handler := range httpHandlers { - handler(r) + for _, handler := range HttpHandlers { + handler.(func(*mux.Router))(r) } port := servercfg.GetAPIPort() diff --git a/ee/initialize.go b/ee/initialize.go index ad659af50..e808d0085 100644 --- a/ee/initialize.go +++ b/ee/initialize.go @@ -19,8 +19,8 @@ func InitEE() { setIsEnterprise() servercfg.Is_EE = true models.SetLogo(retrieveEELogo()) - controller.FullHttpHandlers = append( - controller.FullHttpHandlers, + controller.HttpHandlers = append( + controller.HttpHandlers, ee_controllers.MetricHandlers, ee_controllers.NetworkUsersHandlers, ee_controllers.UserGroupsHandlers, diff --git a/main.go b/main.go index 3449fd012..755d0bcd6 100644 --- a/main.go +++ b/main.go @@ -148,7 +148,7 @@ func startControllers(wg *sync.WaitGroup, ctx context.Context) { } } wg.Add(1) - go controller.HandleRESTRequests(wg, ctx, controller.FullHttpHandlers) + go controller.HandleRESTRequests(wg, ctx) } //Run MessageQueue if servercfg.IsMessageQueueBackend() { From 1b736bc41f16caef0f7a81c125d0c03ce2baca95 Mon Sep 17 00:00:00 2001 From: gabrielseibel1 Date: Mon, 31 Jul 2023 15:10:25 -0300 Subject: [PATCH 10/14] Revert "Handle license validation failures with a (re)boot in a limited state" This reverts commit 58cfbbaf522a1345aac1fa67964ebff0a6d60cd8. --- ee/license.go | 38 ++++++++++++++++---------------------- main.go | 8 -------- 2 files changed, 16 insertions(+), 30 deletions(-) diff --git a/ee/license.go b/ee/license.go index 385062b25..dfbb36eca 100644 --- a/ee/license.go +++ b/ee/license.go @@ -12,6 +12,7 @@ import ( "golang.org/x/exp/slog" "io" "net/http" + "os" "time" "github.com/gravitl/netmaker/database" @@ -34,14 +35,7 @@ type apiServerConf struct { // AddLicenseHooks - adds the validation and cache clear hooks func AddLicenseHooks() { logic.HookManagerCh <- models.HookDetails{ - Hook: func() error { - if err := ValidateLicense(); err != nil { - // stop the program when license is not valid anymore - // if the server restarts and still fails the license check, it can reboot in a limited mode - return fmt.Errorf("%w: %s", logic.HookManagerFatalError, err.Error()) - } - return nil - }, + Hook: ValidateLicense, Interval: time.Hour, } logic.HookManagerCh <- models.HookDetails{ @@ -54,26 +48,25 @@ func AddLicenseHooks() { // checks if a license is valid + limits are not exceeded // if license is free_tier and limits exceeds, then server should terminate // if license is not valid, server should terminate -// TODO update comment func ValidateLicense() error { licenseKeyValue := servercfg.GetLicenseKey() netmakerTenantID := servercfg.GetNetmakerTenantID() slog.Info("proceeding with Netmaker license validation...") if len(licenseKeyValue) == 0 { - return wrappedInErrValidation(errors.New("empty license-key (LICENSE_KEY environment variable)")) + failValidation(errors.New("empty license-key (LICENSE_KEY environment variable)")) } if len(netmakerTenantID) == 0 { - return wrappedInErrValidation(errors.New("empty tenant-id (NETMAKER_TENANT_ID environment variable)")) + failValidation(errors.New("empty tenant-id (NETMAKER_TENANT_ID environment variable)")) } apiPublicKey, err := getLicensePublicKey(licenseKeyValue) if err != nil { - return wrappedInErrValidation(fmt.Errorf("failed to get license public key: %w", err)) + failValidation(fmt.Errorf("failed to get license public key: %w", err)) } tempPubKey, tempPrivKey, err := FetchApiServerKeys() if err != nil { - return wrappedInErrValidation(fmt.Errorf("failed to fetch api server keys: %w", err)) + failValidation(fmt.Errorf("failed to fetch api server keys: %w", err)) } licenseSecret := LicenseSecret{ @@ -83,35 +76,35 @@ func ValidateLicense() error { secretData, err := json.Marshal(&licenseSecret) if err != nil { - return wrappedInErrValidation(fmt.Errorf("failed to marshal license secret: %w", err)) + failValidation(fmt.Errorf("failed to marshal license secret: %w", err)) } encryptedData, err := ncutils.BoxEncrypt(secretData, apiPublicKey, tempPrivKey) if err != nil { - return wrappedInErrValidation(fmt.Errorf("failed to encrypt license secret data: %w", err)) + failValidation(fmt.Errorf("failed to encrypt license secret data: %w", err)) } validationResponse, err := validateLicenseKey(encryptedData, tempPubKey) if err != nil { - return wrappedInErrValidation(fmt.Errorf("failed to validate license key: %w", err)) + failValidation(fmt.Errorf("failed to validate license key: %w", err)) } if len(validationResponse) == 0 { - return wrappedInErrValidation(errors.New("empty validation response")) + failValidation(errors.New("empty validation response")) } var licenseResponse ValidatedLicense if err = json.Unmarshal(validationResponse, &licenseResponse); err != nil { - return wrappedInErrValidation(fmt.Errorf("failed to unmarshal validation response: %w", err)) + failValidation(fmt.Errorf("failed to unmarshal validation response: %w", err)) } respData, err := ncutils.BoxDecrypt(base64decode(licenseResponse.EncryptedLicense), apiPublicKey, tempPrivKey) if err != nil { - return wrappedInErrValidation(fmt.Errorf("failed to decrypt license: %w", err)) + failValidation(fmt.Errorf("failed to decrypt license: %w", err)) } license := LicenseKey{} if err = json.Unmarshal(respData, &license); err != nil { - return wrappedInErrValidation(fmt.Errorf("failed to unmarshal license key: %w", err)) + failValidation(fmt.Errorf("failed to unmarshal license key: %w", err)) } slog.Info("License validation succeeded!") @@ -165,8 +158,9 @@ func FetchApiServerKeys() (pub *[32]byte, priv *[32]byte, err error) { return pub, priv, nil } -func wrappedInErrValidation(err error) error { - return fmt.Errorf("%w: %s", ErrValidation, err.Error()) +func failValidation(err error) { + slog.Error(errValidation.Error(), "error", err) + os.Exit(0) } func getLicensePublicKey(licensePubKeyEncoded string) (*[32]byte, error) { diff --git a/main.go b/main.go index 755d0bcd6..e9dab92f6 100644 --- a/main.go +++ b/main.go @@ -124,14 +124,6 @@ func initialize() { // Client Mode Prereq Check } func startControllers(wg *sync.WaitGroup, ctx context.Context) { - // limit the controllers when unlicensed - if servercfg.IsUnlicensed { - wg.Add(2) - go controller.HandleRESTRequests(wg, ctx, controller.LimitedHttpHandlers) - go logic.StartHookManager(ctx, wg) - return - } - if servercfg.IsDNSMode() { err := logic.SetDNS() if err != nil { From 2bef48d1a41855a6a09938dae2a5ecedab2e8a83 Mon Sep 17 00:00:00 2001 From: gabrielseibel1 Date: Mon, 31 Jul 2023 15:10:38 -0300 Subject: [PATCH 11/14] Revert "Mark server as unlicensed when ee checks fail" This reverts commit 77c6dbdd3c9cfa6e7d6becedef6251e8617ae367. --- ee/initialize.go | 9 ++------- logic/server.go | 9 +++------ main.go | 6 +----- 3 files changed, 6 insertions(+), 18 deletions(-) diff --git a/ee/initialize.go b/ee/initialize.go index e808d0085..455ee59df 100644 --- a/ee/initialize.go +++ b/ee/initialize.go @@ -11,7 +11,6 @@ import ( "github.com/gravitl/netmaker/logic" "github.com/gravitl/netmaker/models" "github.com/gravitl/netmaker/servercfg" - "golang.org/x/exp/slog" ) // InitEE - Initialize EE Logic @@ -26,18 +25,14 @@ func InitEE() { ee_controllers.UserGroupsHandlers, ee_controllers.RelayHandlers, ) - logic.EnterpriseCheckFuncs = append(logic.EnterpriseCheckFuncs, func() error { + logic.EnterpriseCheckFuncs = append(logic.EnterpriseCheckFuncs, func() { // == License Handling == - if err := ValidateLicense(); err != nil { - slog.Error(err.Error()) - return err - } + ValidateLicense() logger.Log(0, "proceeding with Paid Tier license") logic.SetFreeTierForTelemetry(false) // == End License Handling == AddLicenseHooks() resetFailover() - return nil }) logic.EnterpriseFailoverFunc = eelogic.SetFailover logic.EnterpriseResetFailoverFunc = eelogic.ResetFailover diff --git a/logic/server.go b/logic/server.go index 39bd1cb25..a8d509326 100644 --- a/logic/server.go +++ b/logic/server.go @@ -6,7 +6,7 @@ import ( ) // EnterpriseCheckFuncs - can be set to run functions for EE -var EnterpriseCheckFuncs []func() error +var EnterpriseCheckFuncs []func() // EnterpriseFailoverFunc - interface to control failover funcs var EnterpriseFailoverFunc func(node *models.Node) error @@ -26,11 +26,8 @@ const KUBERNETES_LISTEN_PORT = 31821 const KUBERNETES_SERVER_MTU = 1024 // EnterpriseCheck - Runs enterprise functions if presented -func EnterpriseCheck() error { +func EnterpriseCheck() { for _, check := range EnterpriseCheckFuncs { - if err := check(); err != nil { - return err - } + check() } - return nil } diff --git a/main.go b/main.go index e9dab92f6..6abd97680 100644 --- a/main.go +++ b/main.go @@ -91,11 +91,7 @@ func initialize() { // Client Mode Prereq Check if err != nil { logger.Log(1, "Timer error occurred: ", err.Error()) } - - if err := logic.EnterpriseCheck(); err != nil { - servercfg.IsUnlicensed = true - return - } + logic.EnterpriseCheck() var authProvider = auth.InitializeAuthProvider() if authProvider != "" { From a7266c76fa0b60340c282d4691682eb4312afef0 Mon Sep 17 00:00:00 2001 From: gabrielseibel1 Date: Mon, 31 Jul 2023 16:21:39 -0300 Subject: [PATCH 12/14] Handle license validation failures with a middleware --- controllers/controller.go | 7 +++++ controllers/server.go | 13 ++++++--- ee/ee_controllers/middleware.go | 18 ++++++++++++ ee/initialize.go | 15 +++++++--- ee/license.go | 51 +++++++++++++++++++++------------ logic/timer.go | 12 +++++--- servercfg/serverconf.go | 6 ++-- 7 files changed, 89 insertions(+), 33 deletions(-) create mode 100644 ee/ee_controllers/middleware.go diff --git a/controllers/controller.go b/controllers/controller.go index 7abef20b3..8e54d38b1 100644 --- a/controllers/controller.go +++ b/controllers/controller.go @@ -14,6 +14,9 @@ import ( "github.com/gravitl/netmaker/servercfg" ) +// HttpMiddlewares - middleware functions for REST interactions +var HttpMiddlewares []mux.MiddlewareFunc + // HttpHandlers - handler functions for REST interactions var HttpHandlers = []interface{}{ nodeHandlers, @@ -42,6 +45,10 @@ func HandleRESTRequests(wg *sync.WaitGroup, ctx context.Context) { originsOk := handlers.AllowedOrigins(strings.Split(servercfg.GetAllowedOrigin(), ",")) methodsOk := handlers.AllowedMethods([]string{http.MethodGet, http.MethodPut, http.MethodPost, http.MethodDelete}) + for _, middleware := range HttpMiddlewares { + r.Use(middleware) + } + for _, handler := range HttpHandlers { handler.(func(*mux.Router))(r) } diff --git a/controllers/server.go b/controllers/server.go index ba4a18f67..79c8e21f0 100644 --- a/controllers/server.go +++ b/controllers/server.go @@ -69,15 +69,20 @@ func getUsage(w http.ResponseWriter, r *http.Request) { // 200: serverConfigResponse func getStatus(w http.ResponseWriter, r *http.Request) { type status struct { - DB bool `json:"db_connected"` - Broker bool `json:"broker_connected"` - UnlicensedEE bool `json:"unlicensed_ee"` + DB bool `json:"db_connected"` + Broker bool `json:"broker_connected"` + LicenseError string `json:"license_error"` + } + + licenseErr := "" + if servercfg.ErrLicenseValidation != nil { + licenseErr = servercfg.ErrLicenseValidation.Error() } currentServerStatus := status{ DB: database.IsConnected(), Broker: mq.IsConnected(), - UnlicensedEE: servercfg.Is_EE && servercfg.IsUnlicensed, + LicenseError: licenseErr, } w.Header().Set("Content-Type", "application/json") diff --git a/ee/ee_controllers/middleware.go b/ee/ee_controllers/middleware.go new file mode 100644 index 000000000..35b79ec4c --- /dev/null +++ b/ee/ee_controllers/middleware.go @@ -0,0 +1,18 @@ +package ee_controllers + +import ( + "github.com/gravitl/netmaker/logic" + "github.com/gravitl/netmaker/servercfg" + "net/http" + "strings" +) + +func OnlyServerAPIWhenUnlicensedMiddleware(handler http.Handler) http.Handler { + return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if servercfg.ErrLicenseValidation != nil && !strings.HasPrefix(request.URL.Path, "/api/server") { + logic.ReturnErrorResponse(writer, request, logic.FormatError(servercfg.ErrLicenseValidation, "unauthorized")) + return + } + handler.ServeHTTP(writer, request) + }) +} diff --git a/ee/initialize.go b/ee/initialize.go index 455ee59df..e779147bf 100644 --- a/ee/initialize.go +++ b/ee/initialize.go @@ -7,10 +7,10 @@ import ( controller "github.com/gravitl/netmaker/controllers" "github.com/gravitl/netmaker/ee/ee_controllers" eelogic "github.com/gravitl/netmaker/ee/logic" - "github.com/gravitl/netmaker/logger" "github.com/gravitl/netmaker/logic" "github.com/gravitl/netmaker/models" "github.com/gravitl/netmaker/servercfg" + "golang.org/x/exp/slog" ) // InitEE - Initialize EE Logic @@ -18,6 +18,10 @@ func InitEE() { setIsEnterprise() servercfg.Is_EE = true models.SetLogo(retrieveEELogo()) + controller.HttpMiddlewares = append( + controller.HttpMiddlewares, + ee_controllers.OnlyServerAPIWhenUnlicensedMiddleware, + ) controller.HttpHandlers = append( controller.HttpHandlers, ee_controllers.MetricHandlers, @@ -27,8 +31,11 @@ func InitEE() { ) logic.EnterpriseCheckFuncs = append(logic.EnterpriseCheckFuncs, func() { // == License Handling == - ValidateLicense() - logger.Log(0, "proceeding with Paid Tier license") + if err := ValidateLicense(); err != nil { + slog.Error(err.Error()) + return + } + slog.Info("proceeding with Paid Tier license") logic.SetFreeTierForTelemetry(false) // == End License Handling == AddLicenseHooks() @@ -48,7 +55,7 @@ func resetFailover() { for _, net := range nets { err = eelogic.ResetFailover(net.NetID) if err != nil { - logger.Log(0, "failed to reset failover on network", net.NetID, ":", err.Error()) + slog.Error("failed to reset failover", "network", net.NetID, "error", err.Error()) } } } diff --git a/ee/license.go b/ee/license.go index dfbb36eca..b10d5b1b4 100644 --- a/ee/license.go +++ b/ee/license.go @@ -12,7 +12,6 @@ import ( "golang.org/x/exp/slog" "io" "net/http" - "os" "time" "github.com/gravitl/netmaker/database" @@ -44,29 +43,40 @@ func AddLicenseHooks() { } } -// ValidateLicense - the initial license check for netmaker server +// ValidateLicense - the initial and periodic license check for netmaker server // checks if a license is valid + limits are not exceeded -// if license is free_tier and limits exceeds, then server should terminate -// if license is not valid, server should terminate -func ValidateLicense() error { +// if license is free_tier and limits exceeds, then function should error +// if license is not valid, function should error +func ValidateLicense() (err error) { + defer func() { + if err != nil { + err = fmt.Errorf("%w: %s", errValidation, err.Error()) + servercfg.ErrLicenseValidation = err + } + }() + licenseKeyValue := servercfg.GetLicenseKey() netmakerTenantID := servercfg.GetNetmakerTenantID() slog.Info("proceeding with Netmaker license validation...") if len(licenseKeyValue) == 0 { - failValidation(errors.New("empty license-key (LICENSE_KEY environment variable)")) + err = errors.New("empty license-key (LICENSE_KEY environment variable)") + return err } if len(netmakerTenantID) == 0 { - failValidation(errors.New("empty tenant-id (NETMAKER_TENANT_ID environment variable)")) + err = errors.New("empty tenant-id (NETMAKER_TENANT_ID environment variable)") + return err } apiPublicKey, err := getLicensePublicKey(licenseKeyValue) if err != nil { - failValidation(fmt.Errorf("failed to get license public key: %w", err)) + err = fmt.Errorf("failed to get license public key: %w", err) + return err } tempPubKey, tempPrivKey, err := FetchApiServerKeys() if err != nil { - failValidation(fmt.Errorf("failed to fetch api server keys: %w", err)) + err = fmt.Errorf("failed to fetch api server keys: %w", err) + return err } licenseSecret := LicenseSecret{ @@ -76,35 +86,42 @@ func ValidateLicense() error { secretData, err := json.Marshal(&licenseSecret) if err != nil { - failValidation(fmt.Errorf("failed to marshal license secret: %w", err)) + err = fmt.Errorf("failed to marshal license secret: %w", err) + return err } encryptedData, err := ncutils.BoxEncrypt(secretData, apiPublicKey, tempPrivKey) if err != nil { - failValidation(fmt.Errorf("failed to encrypt license secret data: %w", err)) + err = fmt.Errorf("failed to encrypt license secret data: %w", err) + return err } validationResponse, err := validateLicenseKey(encryptedData, tempPubKey) if err != nil { - failValidation(fmt.Errorf("failed to validate license key: %w", err)) + err = fmt.Errorf("failed to validate license key: %w", err) + return err } if len(validationResponse) == 0 { - failValidation(errors.New("empty validation response")) + err = errors.New("empty validation response") + return err } var licenseResponse ValidatedLicense if err = json.Unmarshal(validationResponse, &licenseResponse); err != nil { - failValidation(fmt.Errorf("failed to unmarshal validation response: %w", err)) + err = fmt.Errorf("failed to unmarshal validation response: %w", err) + return err } respData, err := ncutils.BoxDecrypt(base64decode(licenseResponse.EncryptedLicense), apiPublicKey, tempPrivKey) if err != nil { - failValidation(fmt.Errorf("failed to decrypt license: %w", err)) + err = fmt.Errorf("failed to decrypt license: %w", err) + return err } license := LicenseKey{} if err = json.Unmarshal(respData, &license); err != nil { - failValidation(fmt.Errorf("failed to unmarshal license key: %w", err)) + err = fmt.Errorf("failed to unmarshal license key: %w", err) + return err } slog.Info("License validation succeeded!") @@ -159,8 +176,6 @@ func FetchApiServerKeys() (pub *[32]byte, priv *[32]byte, err error) { } func failValidation(err error) { - slog.Error(errValidation.Error(), "error", err) - os.Exit(0) } func getLicensePublicKey(licensePubKeyEncoded string) (*[32]byte, error) { diff --git a/logic/timer.go b/logic/timer.go index 89070f2fe..7a21a3785 100644 --- a/logic/timer.go +++ b/logic/timer.go @@ -3,10 +3,11 @@ package logic import ( "context" "fmt" + "github.com/gravitl/netmaker/logger" + "golang.org/x/exp/slog" "sync" "time" - "github.com/gravitl/netmaker/logger" "github.com/gravitl/netmaker/models" ) @@ -52,7 +53,7 @@ func StartHookManager(ctx context.Context, wg *sync.WaitGroup) { for { select { case <-ctx.Done(): - logger.Log(0, "## Stopping Hook Manager") + slog.Error("## Stopping Hook Manager") return case newhook := <-HookManagerCh: wg.Add(1) @@ -70,7 +71,9 @@ func addHookWithInterval(ctx context.Context, wg *sync.WaitGroup, hook func() er case <-ctx.Done(): return case <-ticker.C: - hook() + if err := hook(); err != nil { + slog.Error(err.Error()) + } } } @@ -85,6 +88,7 @@ var timeHooks = []interface{}{ } func loggerDump() error { + // TODO use slog? logger.DumpFile(fmt.Sprintf("data/netmaker.log.%s", time.Now().Format(logger.TimeFormatDay))) return nil } @@ -93,7 +97,7 @@ func loggerDump() error { func runHooks() { for _, hook := range timeHooks { if err := hook.(func() error)(); err != nil { - logger.Log(1, "error occurred when running timer function:", err.Error()) + slog.Error("error occurred when running timer function", "error", err.Error()) } } } diff --git a/servercfg/serverconf.go b/servercfg/serverconf.go index 101ba94f5..f4e70030e 100644 --- a/servercfg/serverconf.go +++ b/servercfg/serverconf.go @@ -18,9 +18,9 @@ import ( const EmqxBrokerType = "emqx" var ( - Version = "dev" - Is_EE = false - IsUnlicensed = false + Version = "dev" + Is_EE = false + ErrLicenseValidation error ) // SetHost - sets the host ip From ec6f6dbb7fa04ffaf8fb663409ec52262792c57e Mon Sep 17 00:00:00 2001 From: gabrielseibel1 Date: Tue, 1 Aug 2023 08:33:33 -0300 Subject: [PATCH 13/14] Forbid responses if unlicensed ee and not in status api --- ee/ee_controllers/middleware.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ee/ee_controllers/middleware.go b/ee/ee_controllers/middleware.go index 35b79ec4c..dc853524b 100644 --- a/ee/ee_controllers/middleware.go +++ b/ee/ee_controllers/middleware.go @@ -4,13 +4,12 @@ import ( "github.com/gravitl/netmaker/logic" "github.com/gravitl/netmaker/servercfg" "net/http" - "strings" ) func OnlyServerAPIWhenUnlicensedMiddleware(handler http.Handler) http.Handler { return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - if servercfg.ErrLicenseValidation != nil && !strings.HasPrefix(request.URL.Path, "/api/server") { - logic.ReturnErrorResponse(writer, request, logic.FormatError(servercfg.ErrLicenseValidation, "unauthorized")) + if servercfg.ErrLicenseValidation != nil && request.URL.Path != "/api/server/status" { + logic.ReturnErrorResponse(writer, request, logic.FormatError(servercfg.ErrLicenseValidation, "forbidden")) return } handler.ServeHTTP(writer, request) From c04b5a9e5d3b001b4138ab10378ad7f9feddc204 Mon Sep 17 00:00:00 2001 From: gabrielseibel1 Date: Tue, 1 Aug 2023 13:56:34 -0300 Subject: [PATCH 14/14] Remove unused func --- ee/license.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/ee/license.go b/ee/license.go index b10d5b1b4..5f05df432 100644 --- a/ee/license.go +++ b/ee/license.go @@ -175,9 +175,6 @@ func FetchApiServerKeys() (pub *[32]byte, priv *[32]byte, err error) { return pub, priv, nil } -func failValidation(err error) { -} - func getLicensePublicKey(licensePubKeyEncoded string) (*[32]byte, error) { decodedPubKey := base64decode(licensePubKeyEncoded) return ncutils.ConvertBytesToKey(decodedPubKey)