feature: 支持上传未压缩的镜像层

moby_distribution/__init__.py

@@ -7,7 +7,7 @@
 from moby_distribution.spec.image_json import ImageJSON
 from moby_distribution.spec.manifest import ManifestSchema1, ManifestSchema2, OCIManifestSchema1
 
-__version__ = "0.5.11"
+__version__ = "0.6.0"
 __ALL__ = [
     "DockerRegistryV2Client",
     "Blob",

moby_distribution/registry/resources/image.py

@@ -132,6 +132,7 @@ def from_tarball(
 
             layers = []
             for layer in manifest.Layers:
+                # gzip it for smaller size
                 gzipped_filepath = workplace / (layer + ".gz")
                 with (workplace / layer).open(mode="rb") as fh, gzip.open(gzipped_filepath, mode="wb") as compressed:
                     shutil.copyfileobj(fh, compressed)
@@ -226,24 +227,28 @@ def add_layer(self, layer: LayerRef, history: Optional[History] = None) -> Docke
         uncompressed_tarball_signer = HashSignWrapper()
         # Add local layer
         if layer.local_path:
-            # Step 1: calculate the sha256 sum for the gzipped_tarball
-            gzipped_signer = HashSignWrapper()
+            # Step 1: calculate the sha256 sum for the tarball file
+            raw_tarball_signer = HashSignWrapper()
             with layer.local_path.open(mode="rb") as gzipped:
-                shutil.copyfileobj(gzipped, gzipped_signer)
-                size = gzipped_signer.tell()
+                shutil.copyfileobj(gzipped, raw_tarball_signer)
+                size = raw_tarball_signer.tell()
 
             # Step 2: calculate the sha256 sum for the uncompressed_tarball
-            with gzip.open(filename=layer.local_path) as uncompressed:
-                shutil.copyfileobj(uncompressed, uncompressed_tarball_signer)
+            # for gzipped tarball, we need decompress first
+            try:
+                with gzip.open(filename=layer.local_path) as uncompressed:
+                    shutil.copyfileobj(uncompressed, uncompressed_tarball_signer)
+            except OSError:
+                uncompressed_tarball_signer = raw_tarball_signer
 
-            if layer.digest and layer.digest != gzipped_signer.digest():
+            if layer.digest and layer.digest != raw_tarball_signer.digest():
                 raise ValueError(
                     "Wrong digest, layer.digest<'%s'> != signer.digest<'%s'>",
                     layer.digest,
-                    gzipped_signer.digest(),
+                    raw_tarball_signer.digest(),
                 )
 
-            layer.digest = gzipped_signer.digest()
+            layer.digest = raw_tarball_signer.digest()
             layer.repo = self.repo
             layer.size = size
 
@@ -252,21 +257,23 @@ def add_layer(self, layer: LayerRef, history: Optional[History] = None) -> Docke
             with generate_temp_dir() as temp_dir:
                 # Step 1: calculate the sha256 sum for the gzipped_tarball
                 with (temp_dir / "blob").open(mode="wb") as fh:
-                    gzipped_signer = HashSignWrapper(fh=fh)
-                    Blob(repo=layer.repo, digest=layer.digest, fileobj=gzipped_signer, client=self.client).download()
-                    size = gzipped_signer.tell()
+                    raw_tarball_signer = HashSignWrapper(fh=fh)
+                    Blob(
+                        repo=layer.repo, digest=layer.digest, fileobj=raw_tarball_signer, client=self.client
+                    ).download()
+                    size = raw_tarball_signer.tell()
 
                 # Step 2: calculate the sha256 sum for the uncompressed_tarball
                 with gzip.open(filename=(temp_dir / "blob")) as uncompressed:
                     shutil.copyfileobj(uncompressed, uncompressed_tarball_signer)
 
             if layer.size != size:
                 raise ValueError("Wrong Size, layer.size<'%d'> != signer.size<'%d'>", layer.size, size)
-            if layer.digest != gzipped_signer.digest():
+            if layer.digest != raw_tarball_signer.digest():
                 raise ValueError(
                     "Wrong digest, layer.digest<'%s'> != signer.digest<'%s'>",
                     layer.digest,
-                    gzipped_signer.digest(),
+                    raw_tarball_signer.digest(),
                 )
 
         self._dirty = True
@@ -283,7 +290,7 @@ def add_layer(self, layer: LayerRef, history: Optional[History] = None) -> Docke
         self.layers.append(layer)
 
         return DockerManifestLayerDescriptor(
-            digest=gzipped_signer.digest(),
+            digest=raw_tarball_signer.digest(),
             size=size,
         )
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "moby-distribution"
-version = "0.5.11"
+version = "0.6.0"
 description = "Yet another moby(docker) distribution implement by python."
 authors = ["shabbywu <[email protected]>"]
 license = "Apache-2.0"

tests/integration/resources/conftest.py

@@ -5,7 +5,7 @@
 
 import pytest
 
-asserts = Path(__file__).parent / "asserts"
+assets = Path(__file__).parent / "assets"
 
 
 @pytest.fixture
@@ -30,29 +30,34 @@ def temp_reference() -> str:
 
 @pytest.fixture
 def registry_manifest_schema1():
-    return json.loads((asserts / "registry_manifest_schema1.json").read_text())
+    return json.loads((assets / "registry_manifest_schema1.json").read_text())
 
 
 @pytest.fixture
 def registry_manifest_schema2():
-    return json.loads((asserts / "registry_manifest_schema2.json").read_text())
+    return json.loads((assets / "registry_manifest_schema2.json").read_text())
 
 
 @pytest.fixture
 def registry_manifest_schema1_metadata():
-    return json.loads((asserts / "registry_manifest_schema1_metadata.json").read_text())
+    return json.loads((assets / "registry_manifest_schema1_metadata.json").read_text())
 
 
 @pytest.fixture
 def registry_manifest_schema2_metadata():
-    return json.loads((asserts / "registry_manifest_schema2_metadata.json").read_text())
+    return json.loads((assets / "registry_manifest_schema2_metadata.json").read_text())
 
 
 @pytest.fixture
 def alpine_tar():
-    return asserts / "alpine.tar"
+    return assets / "alpine.tar"
+
+
+@pytest.fixture
+def alpine_append_gzip_layer():
+    return assets / "append.tar.gz"
 
 
 @pytest.fixture
 def alpine_append_layer():
-    return asserts / "append.tar.gz"
+    return assets / "append.tar"

tests/integration/resources/test_image.py

@@ -139,3 +139,20 @@ def test_append(self, registry_client, tmp_path, alpine_tar, alpine_append_layer
             docker_cli.containers.run(f"{registry_netloc}/alpine:append", command="cat /append/content", remove=True)
             == b"__flag__\n"
         )
+
+    def test_append_gzip(
+        self, registry_client, tmp_path, alpine_tar, alpine_append_gzip_layer, docker_cli, registry_netloc
+    ):
+        ref = ImageRef.from_tarball(
+            workplace=tmp_path, src=alpine_tar, to_repo="alpine", to_reference="append-gzip", client=registry_client
+        )
+        ref.add_layer(LayerRef(local_path=alpine_append_gzip_layer))
+        ref.push()
+
+        ImageRef.from_image(from_repo="alpine", from_reference="append-gzip", client=registry_client)
+        assert (
+            docker_cli.containers.run(
+                f"{registry_netloc}/alpine:append-gzip", command="cat /append/content", remove=True
+            )
+            == b"__flag__\n"
+        )

tests/spec/conftest.py

@@ -3,29 +3,29 @@
 
 import pytest
 
-asserts = Path(__file__).parent / "asserts"
+assets = Path(__file__).parent / "assets"
 
 
 @pytest.fixture
 def docker_manifest_schema1_dict():
-    return json.loads((asserts / "docker_manifest_schema1.json").read_text())
+    return json.loads((assets / "docker_manifest_schema1.json").read_text())
 
 
 @pytest.fixture
 def docker_manifest_schema2_dict():
-    return json.loads((asserts / "docker_manifest_schema2.json").read_text())
+    return json.loads((assets / "docker_manifest_schema2.json").read_text())
 
 
 @pytest.fixture
 def oci_manifest_schema1_dict():
-    return json.loads((asserts / "oci_manifest_schema1.json").read_text())
+    return json.loads((assets / "oci_manifest_schema1.json").read_text())
 
 
 @pytest.fixture
 def auth_response():
-    return json.loads((asserts / "auth_response.json").read_text())
+    return json.loads((assets / "auth_response.json").read_text())
 
 
 @pytest.fixture
 def image_json_dict():
-    return json.loads((asserts / "image_json.json").read_text())
+    return json.loads((assets / "image_json.json").read_text())