diff --git a/returnn/frontend/container.py b/returnn/frontend/container.py
index c2dd15486a..83704c1cc7 100644
--- a/returnn/frontend/container.py
+++ b/returnn/frontend/container.py
@@ -84,7 +84,7 @@ def __setitem__(self, idx: Union[slice, int], module: Union[__ModT, Iterable[__M
             # Delete also remaining, and then re-add them later, such that indices are correct.
             for k, _ in mod_items[idx.start :]:
                 delattr(self, k)
-            i = idx.start
+            i = idx.start or 0
             for mod_ in module:
                 assert not hasattr(self, str(i))
                 setattr(self, str(i), _convert_to_module(mod_))