More tolerant to crashes and failures

thatguy11325 · Oct 13, 2024 · f106574 · f106574
1 parent d803cc4
commit f106574
Showing 1 changed file with 13 additions and 9 deletions.
diff --git a/pokemonred_puffer/sweep.py b/pokemonred_puffer/sweep.py
@@ -131,21 +131,25 @@ def launch_sweep(
                 elif (
                     run["state"]
                     in [
-                        # RunState.failed.value,
+                        RunState.failed.value,
                         RunState.finished.value,
-                        # RunState.crashed.value,
+                        RunState.crashed.value,
                     ]
                     and run["name"] not in finished
                 ):
                     finished.add(run["name"])
                     summary_metrics = json.loads(run["summaryMetrics"])
-                    obs_in = ObservationInParam(
-                        input=json.loads(run["config"])["x"]["value"],
-                        # TODO: try out other stats like required count
-                        output=summary_metrics["environment/stats/event"],
-                        cost=summary_metrics["performance/uptime"],
-                    )
-                    carbs.observe(obs_in)
+                    if (
+                        "environment/stats/event" in summary_metrics
+                        and "performance/uptime" in summary_metrics
+                    ):
+                        obs_in = ObservationInParam(
+                            input=json.loads(run["config"])["x"]["value"],
+                            # TODO: try out other stats like required count
+                            output=summary_metrics["environment/stats/event"],
+                            cost=summary_metrics["performance/uptime"],
+                        )
+                        carbs.observe(obs_in)
                 elif run["state"] == RunState.pending:
                     print(f"PENDING RUN FOUND {run['name']}")
         sweep.print_status()