Skip to content

Commit

Permalink
[core] Ensure AutoEnv SOR failure goes to ERROR
Browse files Browse the repository at this point in the history
  • Loading branch information
teo committed Mar 4, 2024
1 parent 78894d7 commit 4af5139
Showing 1 changed file with 39 additions and 0 deletions.
39 changes: 39 additions & 0 deletions core/environment/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -937,6 +937,7 @@ func (envs *Manager) CreateAutoEnvironment(workflowPath string, userVars map[str
log.WithField("partition", env.Id().String()).
WithField("state", envState).
Debug("could not transition to ERROR after failed deployment/configuration, cleanup in progress")

}

envTasks := env.Workflow().GetTasks()
Expand Down Expand Up @@ -973,7 +974,45 @@ func (envs *Manager) CreateAutoEnvironment(workflowPath string, userVars map[str

err = env.TryTransition(trans)
if err != nil {
envState := env.CurrentState()

env.sendEnvironmentEvent(&event.EnvironmentEvent{EnvironmentID: env.Id().String(), Error: err})

log.WithField("state", envState).
WithField("environment", env.Id().String()).
WithError(err).
Warn("environment start activity failed, cleanup in progress")

err = env.TryTransition(NewGoErrorTransition(
envs.taskman),
)
if err != nil {
log.WithField("partition", env.Id().String()).
WithField("state", envState).
Debug("could not transition to ERROR after failed start activity, cleanup in progress")

env.setState("ERROR")
}

envTasks := env.Workflow().GetTasks()
// TeardownEnvironment manages the envs.mu internally
err = envs.TeardownEnvironment(env.Id(), true /*force*/)
if err != nil {
env.sendEnvironmentEvent(&event.EnvironmentEvent{EnvironmentID: env.Id().String(), Error: err})
}

killedTasks, _, rlsErr := envs.taskman.KillTasks(envTasks.GetTaskIds())
if rlsErr != nil {
log.WithError(rlsErr).Warn("task teardown error")
}
log.WithFields(logrus.Fields{
"killedCount": len(killedTasks),
"lastEnvState": envState,
"level": infologger.IL_Support,
"partition": env.Id().String(),
}).Info("environment start activity failed, tasks were cleaned up")

log.WithField("partition", env.Id().String()).Info("environment teardown complete")
return
}

Expand Down

0 comments on commit 4af5139

Please sign in to comment.