From 27208abc50b22f0cda0aefeda5e5e116b202802a Mon Sep 17 00:00:00 2001 From: Dan Allan Date: Thu, 25 Apr 2024 13:59:27 -0400 Subject: [PATCH 1/5] Give blosc2 distinct content-encoding --- docs/source/explanations/compression.md | 2 +- tiled/_tests/test_decoders.py | 6 +++--- tiled/client/decoders.py | 4 ++-- tiled/media_type_registration.py | 2 +- tiled/server/metrics.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/source/explanations/compression.md b/docs/source/explanations/compression.md index 2603861bb..9f56ed661 100644 --- a/docs/source/explanations/compression.md +++ b/docs/source/explanations/compression.md @@ -50,7 +50,7 @@ entries in this table above later ones. | Method | Accept-Encoding | Required Python Package | | ---------------------------------------------------------------- | --------------- | ----------------------- | -| [blosc](https://www.blosc.org/) | `blosc` | `blosc` | +| [blosc2](https://www.blosc.org/) | `blosc2` | `blosc2` | | [lz4](https://en.wikipedia.org/wiki/LZ4_(compression_algorithm)) | `lz4` | `lz4` | | [Zstandard](https://facebook.github.io/zstd/) | `zstd` | `zstandard` | | [gzip](https://en.wikipedia.org/wiki/Gzip) | `gzip` | none (built in) | diff --git a/tiled/_tests/test_decoders.py b/tiled/_tests/test_decoders.py index 2b99113f6..47ff36860 100644 --- a/tiled/_tests/test_decoders.py +++ b/tiled/_tests/test_decoders.py @@ -31,11 +31,11 @@ def test_zstd(client): assert "zstd" in response.headers["Content-Encoding"] -def test_blosc(client): +def test_blosc2(client): ac = client["compresses_well"] with record_history() as h: ac[:] (response,) = h.responses (request,) = h.requests - assert "blosc" in request.headers["Accept-Encoding"] - assert "blosc" in response.headers["Content-Encoding"] + assert "blosc2" in request.headers["Accept-Encoding"] + assert "blosc2" in response.headers["Content-Encoding"] diff --git a/tiled/client/decoders.py b/tiled/client/decoders.py index 02617a245..d54264be0 100644 --- a/tiled/client/decoders.py +++ b/tiled/client/decoders.py @@ -5,7 +5,7 @@ if modules_available("blosc2"): - class BloscDecoder: + class Blosc2Decoder: def __init__(self): # Blosc seems to have no streaming interface. # Accumulate response data in a cache here, @@ -26,7 +26,7 @@ def flush(self) -> bytes: data = b"".join(self._data) return blosc2.decompress(data) - SUPPORTED_DECODERS["blosc"] = BloscDecoder + SUPPORTED_DECODERS["blosc2"] = Blosc2Decoder if modules_available("zstandard"): diff --git a/tiled/media_type_registration.py b/tiled/media_type_registration.py index a3e80d6f9..854f20699 100644 --- a/tiled/media_type_registration.py +++ b/tiled/media_type_registration.py @@ -355,4 +355,4 @@ def close(self): pass for media_type in ["application/octet-stream", APACHE_ARROW_FILE_MIME_TYPE]: - compression_registry.register(media_type, "blosc", BloscBuffer) + compression_registry.register(media_type, "blosc2", BloscBuffer) diff --git a/tiled/server/metrics.py b/tiled/server/metrics.py index 29750d9c2..2139fe132 100644 --- a/tiled/server/metrics.py +++ b/tiled/server/metrics.py @@ -88,7 +88,7 @@ READ_DURATION.labels(code=code, method="GET", endpoint=endpoint) TOKENIZE_DURATION.labels(code=code, method="GET", endpoint=endpoint) PACK_DURATION.labels(code=code, method="GET", endpoint=endpoint) - for encoding in ["blosc", "gzip", "lz4", "zstd"]: + for encoding in ["blosc2", "gzip", "lz4", "zstd"]: COMPRESSION_DURATION.labels( code=code, method="GET", endpoint=endpoint, encoding=encoding ) From 5974074aa19109e17e4454bb5f8a0baca8c33e2d Mon Sep 17 00:00:00 2001 From: Dan Allan Date: Thu, 25 Apr 2024 15:00:29 -0400 Subject: [PATCH 2/5] Update CHANGELOG --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee48592a0..f693eff75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,14 @@ Write the date in place of the "Unreleased" in the case a new version is release # Changelog +## Unreleased + +### Fixed + +- The `content-encoding` `blosc` was recently upgraded from Blosc to Blsoc2. + The `content-encoding` has been renamed to `blosc2` to avoid version + confusion between different versions of Tiled servers and clients. + ## v0.1.0a119 (24 April 2024) ### Fixed From bee075e14d11e10a0243a89f7cc39e7b53becb80 Mon Sep 17 00:00:00 2001 From: Dan Allan Date: Thu, 25 Apr 2024 15:23:18 -0400 Subject: [PATCH 3/5] Client now supports blosc2, zstd. --- docs/source/explanations/compression.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/explanations/compression.md b/docs/source/explanations/compression.md index 9f56ed661..26cb9de60 100644 --- a/docs/source/explanations/compression.md +++ b/docs/source/explanations/compression.md @@ -55,8 +55,8 @@ entries in this table above later ones. | [Zstandard](https://facebook.github.io/zstd/) | `zstd` | `zstandard` | | [gzip](https://en.wikipedia.org/wiki/Gzip) | `gzip` | none (built in) | -The Tiled Python *client* currently supports gzip and blosc (if the Python -package `blosc` is installed). +The Tiled Python *client* currently supports gzip, zstd, and blosc2 (as long as +the associated optional dependency is installed). ## Example Requests and Responses From da73efed7fd7e457e475d0b66dcbaff9eed7962a Mon Sep 17 00:00:00 2001 From: Dan Allan Date: Thu, 25 Apr 2024 15:23:33 -0400 Subject: [PATCH 4/5] Update blosc -> blosc2 in example logs --- docs/source/how-to/client-logger.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/source/how-to/client-logger.md b/docs/source/how-to/client-logger.md index acb84815d..0aeba57c1 100644 --- a/docs/source/how-to/client-logger.md +++ b/docs/source/how-to/client-logger.md @@ -25,22 +25,22 @@ Requests (`->`) and responses (`<-`) will now be logged to the console, like so. ```py >>> c = from_uri("https://tiled-demo.blueskyproject.io") -16:49:22.307 -> GET 'https://tiled-demo.blueskyproject.io/?root_path=true' 'host:tiled-demo.blueskyproject.io' 'accept:*/*' 'accept-encoding:gzip,blosc' 'connection:keep-alive' 'user-agent:python-tiled/0.1.0a49' +16:49:22.307 -> GET 'https://tiled-demo.blueskyproject.io/?root_path=true' 'host:tiled-demo.blueskyproject.io' 'accept:*/*' 'accept-encoding:gzip,blosc2' 'connection:keep-alive' 'user-agent:python-tiled/0.1.0a49' 16:49:22.486 <- 200 server:nginx/1.18.0 (Ubuntu) date:Tue, 01 Feb 2022 21:49:22 GMT content-type:application/json content-length:761 connection:keep-alive etag:35b70c6412c39db8b7b5132ddf61973c expires:Tue, 01 Feb 2022 21:59:22 GMT content-encoding:gzip vary:Accept-Encoding server-timing:tok;dur=0.1, pack;dur=0.0, compress;dur=0.1;ratio=3.1, app;dur=3.9 set-cookie:tiled_csrf=-fyaLez0YkradgcEVYBJh4QotR5MNyzouV0SV0NWHmM; HttpOnly; Path=/; SameSite=lax -16:49:22.492 -> GET 'https://tiled-demo.blueskyproject.io/?root_path=true' 'host:tiled-demo.blueskyproject.io' 'accept:application/x-msgpack' 'accept-encoding:gzip,blosc' 'connection:keep-alive' 'user-agent:python-tiled/0.1.0a49' 'cookie:tiled_csrf=-fyaLez0YkradgcEVYBJh4QotR5MNyzouV0SV0NWHmM' +16:49:22.492 -> GET 'https://tiled-demo.blueskyproject.io/?root_path=true' 'host:tiled-demo.blueskyproject.io' 'accept:application/x-msgpack' 'accept-encoding:gzip,blosc2' 'connection:keep-alive' 'user-agent:python-tiled/0.1.0a49' 'cookie:tiled_csrf=-fyaLez0YkradgcEVYBJh4QotR5MNyzouV0SV0NWHmM' 16:49:22.531 <- 200 server:nginx/1.18.0 (Ubuntu) date:Tue, 01 Feb 2022 21:49:22 GMT content-type:application/x-msgpack content-length:773 connection:keep-alive etag:35b70c6412c39db8b7b5132ddf61973c expires:Tue, 01 Feb 2022 21:59:22 GMT content-encoding:gzip vary:Accept-Encoding server-timing:tok;dur=0.1, pack;dur=0.0, compress;dur=0.1;ratio=2.7, app;dur=4.5 -16:49:22.535 -> GET 'https://tiled-demo.blueskyproject.io//metadata/' 'host:tiled-demo.blueskyproject.io' 'accept:application/x-msgpack' 'accept-encoding:gzip,blosc' 'connection:keep-alive' 'user-agent:python-tiled/0.1.0a49' 'cookie:tiled_csrf=-fyaLez0YkradgcEVYBJh4QotR5MNyzouV0SV0NWHmM' +16:49:22.535 -> GET 'https://tiled-demo.blueskyproject.io//metadata/' 'host:tiled-demo.blueskyproject.io' 'accept:application/x-msgpack' 'accept-encoding:gzip,blosc2' 'connection:keep-alive' 'user-agent:python-tiled/0.1.0a49' 'cookie:tiled_csrf=-fyaLez0YkradgcEVYBJh4QotR5MNyzouV0SV0NWHmM' 16:49:22.572 <- 200 server:nginx/1.18.0 (Ubuntu) date:Tue, 01 Feb 2022 21:49:22 GMT content-type:application/x-msgpack content-length:292 connection:keep-alive etag:821dd2a8b431ecd016f94cacd44af74f server-timing:tok;dur=0.0, pack;dur=0.0, app;dur=3.8 >>> t = c['generated']['short_table'] -16:58:22.589 -> GET 'https://tiled-demo.blueskyproject.io/search/?filter%5Blookup%5D%5Bcondition%5D%5Bkey%5D=generated&sort=' 'host:tiled-demo.blueskyproject.io' 'accept:application/x-msgpack' 'accept-encoding:gzip,blosc' 'connection:keep-alive' 'user-agent:python-tiled/0.1.0a49.post0.dev0+g6dd1e5f' 'cookie:tiled_csrf=-fyaLez0YkradgcEVYBJh4QotR5MNyzouV0SV0NWHmM' +16:58:22.589 -> GET 'https://tiled-demo.blueskyproject.io/search/?filter%5Blookup%5D%5Bcondition%5D%5Bkey%5D=generated&sort=' 'host:tiled-demo.blueskyproject.io' 'accept:application/x-msgpack' 'accept-encoding:gzip,blosc2' 'connection:keep-alive' 'user-agent:python-tiled/0.1.0a49.post0.dev0+g6dd1e5f' 'cookie:tiled_csrf=-fyaLez0YkradgcEVYBJh4QotR5MNyzouV0SV0NWHmM' 16:58:22.635 <- 200 server:nginx/1.18.0 (Ubuntu) date:Tue, 01 Feb 2022 21:58:22 GMT content-type:application/x-msgpack content-length:502 connection:keep-alive etag:53a7b8a84ec504259a8c29903a25ade0 server-timing:tok;dur=0.0, pack;dur=0.0, app;dur=5.9 -16:58:22.638 -> GET 'https://tiled-demo.blueskyproject.io/search/generated?filter%5Blookup%5D%5Bcondition%5D%5Bkey%5D=short_table&sort=' 'host:tiled-demo.blueskyproject.io' 'accept:application/x-msgpack' 'accept-encoding:gzip,blosc' 'connection:keep-alive' 'user-agent:python-tiled/0.1.0a49.post0.dev0+g6dd1e5f' 'cookie:tiled_csrf=-fyaLez0YkradgcEVYBJh4QotR5MNyzouV0SV0NWHmM' +16:58:22.638 -> GET 'https://tiled-demo.blueskyproject.io/search/generated?filter%5Blookup%5D%5Bcondition%5D%5Bkey%5D=short_table&sort=' 'host:tiled-demo.blueskyproject.io' 'accept:application/x-msgpack' 'accept-encoding:gzip,blosc2' 'connection:keep-alive' 'user-agent:python-tiled/0.1.0a49.post0.dev0+g6dd1e5f' 'cookie:tiled_csrf=-fyaLez0YkradgcEVYBJh4QotR5MNyzouV0SV0NWHmM' 16:58:22.681 <- 200 server:nginx/1.18.0 (Ubuntu) date:Tue, 01 Feb 2022 21:58:22 GMT content-type:application/x-msgpack content-length:944 connection:keep-alive etag:8d81b7891000606ceeb87fa89689c045 content-encoding:gzip vary:Accept-Encoding server-timing:acl;dur=0.0, tok;dur=0.1, pack;dur=0.0, compress;dur=0.1;ratio=4.5, app;dur=12.0 >>> t.read() -16:58:27.134 -> GET 'https://tiled-demo.blueskyproject.io/table/partition/generated/short_table?partition=0' 'host:tiled-demo.blueskyproject.io' 'accept:application/vnd.apache.arrow.file' 'accept-encoding:gzip,blosc' 'connection:keep-alive' 'user-agent:python-tiled/0.1.0a49.post0.dev0+g6dd1e5f' 'cookie:tiled_csrf=-fyaLez0YkradgcEVYBJh4QotR5MNyzouV0SV0NWHmM' -16:58:27.205 <- 200 server:nginx/1.18.0 (Ubuntu) date:Tue, 01 Feb 2022 21:58:27 GMT content-type:application/vnd.apache.arrow.file content-length:3847 connection:keep-alive etag:954688a8ef55915b012bba1e93769710 content-encoding:blosc vary:Accept-Encoding server-timing:acl;dur=0.0, read;dur=1.4, tok;dur=0.2, pack;dur=0.8, compress;dur=0.0;ratio=1.4, app;dur=9.1 +16:58:27.134 -> GET 'https://tiled-demo.blueskyproject.io/table/partition/generated/short_table?partition=0' 'host:tiled-demo.blueskyproject.io' 'accept:application/vnd.apache.arrow.file' 'accept-encoding:gzip,blosc2' 'connection:keep-alive' 'user-agent:python-tiled/0.1.0a49.post0.dev0+g6dd1e5f' 'cookie:tiled_csrf=-fyaLez0YkradgcEVYBJh4QotR5MNyzouV0SV0NWHmM' +16:58:27.205 <- 200 server:nginx/1.18.0 (Ubuntu) date:Tue, 01 Feb 2022 21:58:27 GMT content-type:application/vnd.apache.arrow.file content-length:3847 connection:keep-alive etag:954688a8ef55915b012bba1e93769710 content-encoding:blosc2 vary:Accept-Encoding server-timing:acl;dur=0.0, read;dur=1.4, tok;dur=0.2, pack;dur=0.8, compress;dur=0.0;ratio=1.4, app;dur=9.1 A B C index 0 0.380618 0.761235 1.141853 From 94260fda89f5a317ce5105dcd51690dec0827700 Mon Sep 17 00:00:00 2001 From: Dan Allan Date: Thu, 25 Apr 2024 15:20:13 -0400 Subject: [PATCH 5/5] Fix typo Co-authored-by: Padraic Shafer <76011594+padraic-shafer@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f693eff75..09dd52059 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ Write the date in place of the "Unreleased" in the case a new version is release ### Fixed -- The `content-encoding` `blosc` was recently upgraded from Blosc to Blsoc2. +- The `content-encoding` `blosc` was recently upgraded from Blosc to Blosc2. The `content-encoding` has been renamed to `blosc2` to avoid version confusion between different versions of Tiled servers and clients.