diff --git a/backend/infrahub/core/models.py b/backend/infrahub/core/models.py
index b8aab8d4c3..541c27ba1c 100644
--- a/backend/infrahub/core/models.py
+++ b/backend/infrahub/core/models.py
@@ -31,6 +31,8 @@ def __str__(self) -> str:
 class SchemaBranchDiff(BaseModel):
     nodes: list[str] = Field(default_factory=list)
     generics: list[str] = Field(default_factory=list)
+    removed_nodes: list[str] = Field(default_factory=list)
+    removed_generics: list[str] = Field(default_factory=list)
 
     def to_string(self) -> str:
         return ", ".join(self.nodes + self.generics)
@@ -40,7 +42,7 @@ def to_list(self) -> list[str]:
 
     @property
     def has_diff(self) -> bool:
-        if self.nodes or self.generics:
+        if self.nodes or self.generics or self.removed_nodes or self.removed_generics:
             return True
         return False
 
@@ -50,15 +52,17 @@ class SchemaBranchHash(BaseModel):
     nodes: dict[str, str] = Field(default_factory=dict)
     generics: dict[str, str] = Field(default_factory=dict)
 
-    def compare(self, other: SchemaBranchHash) -> Optional[SchemaBranchDiff]:
+    def compare(self, other: SchemaBranchHash) -> SchemaBranchDiff | None:
         if other.main == self.main:
             return None
 
         return SchemaBranchDiff(
             nodes=[key for key, value in other.nodes.items() if key not in self.nodes or self.nodes[key] != value],
+            removed_nodes=[key for key in self.nodes if key not in other.nodes],
             generics=[
                 key for key, value in other.generics.items() if key not in self.generics or self.generics[key] != value
             ],
+            removed_generics=[key for key in self.generics if key not in other.generics],
         )
 
 
diff --git a/backend/infrahub/core/schema/manager.py b/backend/infrahub/core/schema/manager.py
index b6e04b024d..7f689ad7d0 100644
--- a/backend/infrahub/core/schema/manager.py
+++ b/backend/infrahub/core/schema/manager.py
@@ -636,6 +636,12 @@ async def load_schema_from_db(
                 if filter_value["namespace__values"]:
                     filters[node_type] = filter_value
                     has_filters = True
+            for removed_generic in schema_diff.removed_generics:
+                if removed_generic in schema.generic_names:
+                    schema.delete(name=removed_generic)
+            for removed_node in schema_diff.removed_nodes:
+                if removed_node in schema.node_names:
+                    schema.delete(name=removed_node)
 
         if not has_filters or filters["generics"]:
             generic_schema = self.get(name="SchemaGeneric", branch=branch)
diff --git a/changelog/4836.fixed.md b/changelog/4836.fixed.md
new file mode 100644
index 0000000000..424652878e
--- /dev/null
+++ b/changelog/4836.fixed.md
@@ -0,0 +1 @@
+Ensure that deleted schema nodes are removed from all workers and that the schema is in sync without having to restart