diff --git a/moby_distribution/__init__.py b/moby_distribution/__init__.py index feb62fb..3882250 100644 --- a/moby_distribution/__init__.py +++ b/moby_distribution/__init__.py @@ -7,7 +7,7 @@ from moby_distribution.spec.image_json import ImageJSON from moby_distribution.spec.manifest import ManifestSchema1, ManifestSchema2, OCIManifestSchema1 -__version__ = "0.5.11" +__version__ = "0.6.0" __ALL__ = [ "DockerRegistryV2Client", "Blob", diff --git a/moby_distribution/registry/resources/image.py b/moby_distribution/registry/resources/image.py index 874ef14..c85e8c8 100644 --- a/moby_distribution/registry/resources/image.py +++ b/moby_distribution/registry/resources/image.py @@ -132,6 +132,7 @@ def from_tarball( layers = [] for layer in manifest.Layers: + # gzip it for smaller size gzipped_filepath = workplace / (layer + ".gz") with (workplace / layer).open(mode="rb") as fh, gzip.open(gzipped_filepath, mode="wb") as compressed: shutil.copyfileobj(fh, compressed) @@ -226,24 +227,28 @@ def add_layer(self, layer: LayerRef, history: Optional[History] = None) -> Docke uncompressed_tarball_signer = HashSignWrapper() # Add local layer if layer.local_path: - # Step 1: calculate the sha256 sum for the gzipped_tarball - gzipped_signer = HashSignWrapper() + # Step 1: calculate the sha256 sum for the tarball file + raw_tarball_signer = HashSignWrapper() with layer.local_path.open(mode="rb") as gzipped: - shutil.copyfileobj(gzipped, gzipped_signer) - size = gzipped_signer.tell() + shutil.copyfileobj(gzipped, raw_tarball_signer) + size = raw_tarball_signer.tell() # Step 2: calculate the sha256 sum for the uncompressed_tarball - with gzip.open(filename=layer.local_path) as uncompressed: - shutil.copyfileobj(uncompressed, uncompressed_tarball_signer) + # for gzipped tarball, we need decompress first + try: + with gzip.open(filename=layer.local_path) as uncompressed: + shutil.copyfileobj(uncompressed, uncompressed_tarball_signer) + except OSError: + uncompressed_tarball_signer = raw_tarball_signer - if layer.digest and layer.digest != gzipped_signer.digest(): + if layer.digest and layer.digest != raw_tarball_signer.digest(): raise ValueError( "Wrong digest, layer.digest<'%s'> != signer.digest<'%s'>", layer.digest, - gzipped_signer.digest(), + raw_tarball_signer.digest(), ) - layer.digest = gzipped_signer.digest() + layer.digest = raw_tarball_signer.digest() layer.repo = self.repo layer.size = size @@ -252,9 +257,11 @@ def add_layer(self, layer: LayerRef, history: Optional[History] = None) -> Docke with generate_temp_dir() as temp_dir: # Step 1: calculate the sha256 sum for the gzipped_tarball with (temp_dir / "blob").open(mode="wb") as fh: - gzipped_signer = HashSignWrapper(fh=fh) - Blob(repo=layer.repo, digest=layer.digest, fileobj=gzipped_signer, client=self.client).download() - size = gzipped_signer.tell() + raw_tarball_signer = HashSignWrapper(fh=fh) + Blob( + repo=layer.repo, digest=layer.digest, fileobj=raw_tarball_signer, client=self.client + ).download() + size = raw_tarball_signer.tell() # Step 2: calculate the sha256 sum for the uncompressed_tarball with gzip.open(filename=(temp_dir / "blob")) as uncompressed: @@ -262,11 +269,11 @@ def add_layer(self, layer: LayerRef, history: Optional[History] = None) -> Docke if layer.size != size: raise ValueError("Wrong Size, layer.size<'%d'> != signer.size<'%d'>", layer.size, size) - if layer.digest != gzipped_signer.digest(): + if layer.digest != raw_tarball_signer.digest(): raise ValueError( "Wrong digest, layer.digest<'%s'> != signer.digest<'%s'>", layer.digest, - gzipped_signer.digest(), + raw_tarball_signer.digest(), ) self._dirty = True @@ -283,7 +290,7 @@ def add_layer(self, layer: LayerRef, history: Optional[History] = None) -> Docke self.layers.append(layer) return DockerManifestLayerDescriptor( - digest=gzipped_signer.digest(), + digest=raw_tarball_signer.digest(), size=size, ) diff --git a/pyproject.toml b/pyproject.toml index 94fbf2d..717f3bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "moby-distribution" -version = "0.5.11" +version = "0.6.0" description = "Yet another moby(docker) distribution implement by python." authors = ["shabbywu "] license = "Apache-2.0" diff --git a/tests/integration/resources/asserts/alpine.tar b/tests/integration/resources/assets/alpine.tar similarity index 100% rename from tests/integration/resources/asserts/alpine.tar rename to tests/integration/resources/assets/alpine.tar diff --git a/tests/integration/resources/assets/append.tar b/tests/integration/resources/assets/append.tar new file mode 100644 index 0000000..c8fb5d5 Binary files /dev/null and b/tests/integration/resources/assets/append.tar differ diff --git a/tests/integration/resources/asserts/append.tar.gz b/tests/integration/resources/assets/append.tar.gz similarity index 100% rename from tests/integration/resources/asserts/append.tar.gz rename to tests/integration/resources/assets/append.tar.gz diff --git a/tests/integration/resources/asserts/registry_manifest_schema1.json b/tests/integration/resources/assets/registry_manifest_schema1.json similarity index 100% rename from tests/integration/resources/asserts/registry_manifest_schema1.json rename to tests/integration/resources/assets/registry_manifest_schema1.json diff --git a/tests/integration/resources/asserts/registry_manifest_schema1_metadata.json b/tests/integration/resources/assets/registry_manifest_schema1_metadata.json similarity index 100% rename from tests/integration/resources/asserts/registry_manifest_schema1_metadata.json rename to tests/integration/resources/assets/registry_manifest_schema1_metadata.json diff --git a/tests/integration/resources/asserts/registry_manifest_schema2.json b/tests/integration/resources/assets/registry_manifest_schema2.json similarity index 100% rename from tests/integration/resources/asserts/registry_manifest_schema2.json rename to tests/integration/resources/assets/registry_manifest_schema2.json diff --git a/tests/integration/resources/asserts/registry_manifest_schema2_metadata.json b/tests/integration/resources/assets/registry_manifest_schema2_metadata.json similarity index 100% rename from tests/integration/resources/asserts/registry_manifest_schema2_metadata.json rename to tests/integration/resources/assets/registry_manifest_schema2_metadata.json diff --git a/tests/integration/resources/conftest.py b/tests/integration/resources/conftest.py index b0d6823..ae0ebc8 100644 --- a/tests/integration/resources/conftest.py +++ b/tests/integration/resources/conftest.py @@ -5,7 +5,7 @@ import pytest -asserts = Path(__file__).parent / "asserts" +assets = Path(__file__).parent / "assets" @pytest.fixture @@ -30,29 +30,34 @@ def temp_reference() -> str: @pytest.fixture def registry_manifest_schema1(): - return json.loads((asserts / "registry_manifest_schema1.json").read_text()) + return json.loads((assets / "registry_manifest_schema1.json").read_text()) @pytest.fixture def registry_manifest_schema2(): - return json.loads((asserts / "registry_manifest_schema2.json").read_text()) + return json.loads((assets / "registry_manifest_schema2.json").read_text()) @pytest.fixture def registry_manifest_schema1_metadata(): - return json.loads((asserts / "registry_manifest_schema1_metadata.json").read_text()) + return json.loads((assets / "registry_manifest_schema1_metadata.json").read_text()) @pytest.fixture def registry_manifest_schema2_metadata(): - return json.loads((asserts / "registry_manifest_schema2_metadata.json").read_text()) + return json.loads((assets / "registry_manifest_schema2_metadata.json").read_text()) @pytest.fixture def alpine_tar(): - return asserts / "alpine.tar" + return assets / "alpine.tar" + + +@pytest.fixture +def alpine_append_gzip_layer(): + return assets / "append.tar.gz" @pytest.fixture def alpine_append_layer(): - return asserts / "append.tar.gz" + return assets / "append.tar" diff --git a/tests/integration/resources/test_image.py b/tests/integration/resources/test_image.py index 86588a6..e295c6b 100644 --- a/tests/integration/resources/test_image.py +++ b/tests/integration/resources/test_image.py @@ -139,3 +139,20 @@ def test_append(self, registry_client, tmp_path, alpine_tar, alpine_append_layer docker_cli.containers.run(f"{registry_netloc}/alpine:append", command="cat /append/content", remove=True) == b"__flag__\n" ) + + def test_append_gzip( + self, registry_client, tmp_path, alpine_tar, alpine_append_gzip_layer, docker_cli, registry_netloc + ): + ref = ImageRef.from_tarball( + workplace=tmp_path, src=alpine_tar, to_repo="alpine", to_reference="append-gzip", client=registry_client + ) + ref.add_layer(LayerRef(local_path=alpine_append_gzip_layer)) + ref.push() + + ImageRef.from_image(from_repo="alpine", from_reference="append-gzip", client=registry_client) + assert ( + docker_cli.containers.run( + f"{registry_netloc}/alpine:append-gzip", command="cat /append/content", remove=True + ) + == b"__flag__\n" + ) diff --git a/tests/spec/asserts/auth_response.json b/tests/spec/assets/auth_response.json similarity index 100% rename from tests/spec/asserts/auth_response.json rename to tests/spec/assets/auth_response.json diff --git a/tests/spec/asserts/docker_manifest_schema1.json b/tests/spec/assets/docker_manifest_schema1.json similarity index 100% rename from tests/spec/asserts/docker_manifest_schema1.json rename to tests/spec/assets/docker_manifest_schema1.json diff --git a/tests/spec/asserts/docker_manifest_schema2.json b/tests/spec/assets/docker_manifest_schema2.json similarity index 100% rename from tests/spec/asserts/docker_manifest_schema2.json rename to tests/spec/assets/docker_manifest_schema2.json diff --git a/tests/spec/asserts/image_json.json b/tests/spec/assets/image_json.json similarity index 100% rename from tests/spec/asserts/image_json.json rename to tests/spec/assets/image_json.json diff --git a/tests/spec/asserts/oci_manifest_schema1.json b/tests/spec/assets/oci_manifest_schema1.json similarity index 100% rename from tests/spec/asserts/oci_manifest_schema1.json rename to tests/spec/assets/oci_manifest_schema1.json diff --git a/tests/spec/conftest.py b/tests/spec/conftest.py index f459d77..860d2f4 100644 --- a/tests/spec/conftest.py +++ b/tests/spec/conftest.py @@ -3,29 +3,29 @@ import pytest -asserts = Path(__file__).parent / "asserts" +assets = Path(__file__).parent / "assets" @pytest.fixture def docker_manifest_schema1_dict(): - return json.loads((asserts / "docker_manifest_schema1.json").read_text()) + return json.loads((assets / "docker_manifest_schema1.json").read_text()) @pytest.fixture def docker_manifest_schema2_dict(): - return json.loads((asserts / "docker_manifest_schema2.json").read_text()) + return json.loads((assets / "docker_manifest_schema2.json").read_text()) @pytest.fixture def oci_manifest_schema1_dict(): - return json.loads((asserts / "oci_manifest_schema1.json").read_text()) + return json.loads((assets / "oci_manifest_schema1.json").read_text()) @pytest.fixture def auth_response(): - return json.loads((asserts / "auth_response.json").read_text()) + return json.loads((assets / "auth_response.json").read_text()) @pytest.fixture def image_json_dict(): - return json.loads((asserts / "image_json.json").read_text()) + return json.loads((assets / "image_json.json").read_text())