From e024761aafcb5d9ed74fb0a06eb370826964bd59 Mon Sep 17 00:00:00 2001 From: Nathan Rijksen Date: Fri, 6 Oct 2023 12:46:41 -0700 Subject: [PATCH 1/4] Increase error logging on runtime panics --- pkg/platform/runtime/setup/setup.go | 16 ++++++++++++++++ pkg/platform/runtime/store/store.go | 11 ++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/pkg/platform/runtime/setup/setup.go b/pkg/platform/runtime/setup/setup.go index 5dd8a87023..07d285a291 100644 --- a/pkg/platform/runtime/setup/setup.go +++ b/pkg/platform/runtime/setup/setup.go @@ -2,6 +2,7 @@ package setup import ( "context" + "encoding/json" "errors" "fmt" "net/url" @@ -162,6 +163,21 @@ func New(target Targeter, eventHandler events.Handler, auth *authentication.Auth // Update installs the runtime locally (or updates it if it's already partially installed) func (s *Setup) Update() (rerr error) { + defer func() { + // Panics are serious, and reproducing them in the runtime package is HARD. To help with this we dump + // the build plan when a panic occurs so we have something more to go on. + if r := recover(); r != nil { + // We do a standard error log first here, as rollbar reports will pick up the most recent log lines. + // We can't put the buildplan in the multilog message as it'd be way too big a message for rollbar. + buildplan, err := s.store.BuildPlanRaw() + if err != nil { + logging.Error("Could not get raw buildplan: %s", err) + } + logging.Error("Panic during runtime update: %s, build plan:\n%s", r, buildplan) + multilog.Critical("Panic during runtime update: %s", r) + panic(r) // We're just logging the panic while we have context, we're not meant to handle it here + } + }() defer func() { var ev events.Eventer = events.Success{} if rerr != nil { diff --git a/pkg/platform/runtime/store/store.go b/pkg/platform/runtime/store/store.go index 5bdfd905b5..3239f3e990 100644 --- a/pkg/platform/runtime/store/store.go +++ b/pkg/platform/runtime/store/store.go @@ -247,12 +247,21 @@ func (s *Store) InstallPath() string { return s.installPath } -func (s *Store) BuildPlan() (*bpModel.Build, error) { +func (s *Store) BuildPlanRaw() ([]byte, error) { data, err := fileutils.ReadFile(s.buildPlanFile()) if err != nil { return nil, errs.Wrap(err, "Could not read build plan file.") } + return data, nil +} + +func (s *Store) BuildPlan() (*bpModel.Build, error) { + data, err := s.BuildPlanRaw() + if err != nil { + return nil, errs.Wrap(err, "Could not get build plan file.") + } + var buildPlan bpModel.Build err = json.Unmarshal(data, &buildPlan) if err != nil { From 60996fdd8eb39289ef105236a5bf9d03a79b25b1 Mon Sep 17 00:00:00 2001 From: Nathan Rijksen Date: Fri, 6 Oct 2023 12:48:44 -0700 Subject: [PATCH 2/4] Remove unused import --- pkg/platform/runtime/setup/setup.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/platform/runtime/setup/setup.go b/pkg/platform/runtime/setup/setup.go index 07d285a291..3499efd46a 100644 --- a/pkg/platform/runtime/setup/setup.go +++ b/pkg/platform/runtime/setup/setup.go @@ -2,7 +2,6 @@ package setup import ( "context" - "encoding/json" "errors" "fmt" "net/url" From cfbaac148859608ef6c51741dacd10af9a9a576b Mon Sep 17 00:00:00 2001 From: Nathan Rijksen Date: Fri, 6 Oct 2023 13:27:01 -0700 Subject: [PATCH 3/4] Move comment to correct location --- pkg/platform/runtime/setup/setup.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/platform/runtime/setup/setup.go b/pkg/platform/runtime/setup/setup.go index 3499efd46a..e05d669234 100644 --- a/pkg/platform/runtime/setup/setup.go +++ b/pkg/platform/runtime/setup/setup.go @@ -166,12 +166,12 @@ func (s *Setup) Update() (rerr error) { // Panics are serious, and reproducing them in the runtime package is HARD. To help with this we dump // the build plan when a panic occurs so we have something more to go on. if r := recover(); r != nil { - // We do a standard error log first here, as rollbar reports will pick up the most recent log lines. - // We can't put the buildplan in the multilog message as it'd be way too big a message for rollbar. buildplan, err := s.store.BuildPlanRaw() if err != nil { logging.Error("Could not get raw buildplan: %s", err) } + // We do a standard error log first here, as rollbar reports will pick up the most recent log lines. + // We can't put the buildplan in the multilog message as it'd be way too big a message for rollbar. logging.Error("Panic during runtime update: %s, build plan:\n%s", r, buildplan) multilog.Critical("Panic during runtime update: %s", r) panic(r) // We're just logging the panic while we have context, we're not meant to handle it here From 02b32fe168ff4ef3f7d8c433292086e08eb42e0c Mon Sep 17 00:00:00 2001 From: Nathan Rijksen Date: Fri, 6 Oct 2023 14:06:20 -0700 Subject: [PATCH 4/4] Also debug the env --- pkg/platform/runtime/setup/setup.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/platform/runtime/setup/setup.go b/pkg/platform/runtime/setup/setup.go index e05d669234..fb87535339 100644 --- a/pkg/platform/runtime/setup/setup.go +++ b/pkg/platform/runtime/setup/setup.go @@ -170,9 +170,13 @@ func (s *Setup) Update() (rerr error) { if err != nil { logging.Error("Could not get raw buildplan: %s", err) } + env, err := s.store.EnvDef() + if err != nil { + logging.Error("Could not get envdef: %s", err) + } // We do a standard error log first here, as rollbar reports will pick up the most recent log lines. // We can't put the buildplan in the multilog message as it'd be way too big a message for rollbar. - logging.Error("Panic during runtime update: %s, build plan:\n%s", r, buildplan) + logging.Error("Panic during runtime update: %s, build plan:\n%s\n\nEnvDef:\n%#v", r, buildplan, env) multilog.Critical("Panic during runtime update: %s", r) panic(r) // We're just logging the panic while we have context, we're not meant to handle it here }