From a8656f8ca3a88cc2f5812fd3edaab2f0c0bbdb4c Mon Sep 17 00:00:00 2001 From: Marek Wawrzos Date: Thu, 11 Jul 2024 12:50:35 +0200 Subject: [PATCH 01/23] synthetic image generator --- .../llm_inputs/synthetic_image_generator.py | 43 +++++++++++++++++++ .../tests/test_synthetic_image_generator.py | 33 ++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py create mode 100644 src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py diff --git a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py new file mode 100644 index 000000000..8dd0458e0 --- /dev/null +++ b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py @@ -0,0 +1,43 @@ +import base64 +from enum import Enum, auto +from io import BytesIO + +from genai_perf.exceptions import GenAIPerfException +from PIL import Image + + +class ImageEncoding(Enum): + PIL_IMAGE = auto() + BASE64 = auto() + + +class SyntheticImageGenerator: + def __init__(self, image_encodeing: ImageEncoding): + self.image_encodeing = image_encodeing + self._image_iterator = self.white_images_iterator() + + def __iter__(self): + return self + + def __next__(self): + image = next(self._image_iterator) + return self._encode(image) + + def white_images_iterator(self): + white_image = Image.new("RGB", (100, 100), color="white") + while True: + yield white_image + + def _encode(self, image): + if self.image_encodeing == ImageEncoding.PIL_IMAGE: + return image + elif self.image_encodeing == ImageEncoding.BASE64: + buffered = BytesIO() + image.save(buffered, format="PNG") + data = base64.b64encode(buffered.getvalue()).decode("utf-8") + prefix = "data:image/png;base64" + return f"{prefix},{data}" + else: + raise GenAIPerfException( + f"Unexpected image_encodeing: {self.image_encodeing.name}" + ) diff --git a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py new file mode 100644 index 000000000..a7b594b51 --- /dev/null +++ b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py @@ -0,0 +1,33 @@ +import base64 +from io import BytesIO + +import pytest +from genai_perf.llm_inputs.synthetic_image_generator import ( + ImageEncoding, + SyntheticImageGenerator, +) +from PIL import Image + + +def test_generating_images(): + sut = SyntheticImageGenerator(image_encodeing=ImageEncoding.PIL_IMAGE) + + data = next(sut) + + assert isinstance(data, Image.Image), "generator produces unexpected type of data" + + +def test_base64_encoding(): + sut = SyntheticImageGenerator(image_encodeing=ImageEncoding.BASE64) + + base64String = next(sut) + + base64prefix = "data:image/png;base64," + assert base64String.startswith(base64prefix), "unexpected prefix" + data = base64String[len(base64prefix) :] + + # test if generator encodes to base64 + img_data = base64.b64decode(data) + img_bytes = BytesIO(img_data) + # test if an image is encoded + Image.open(img_bytes) From a5b6dbcdc19da56347349ac9b97143188eea7b69 Mon Sep 17 00:00:00 2001 From: Marek Wawrzos Date: Thu, 11 Jul 2024 13:22:59 +0200 Subject: [PATCH 02/23] format randomization --- .../llm_inputs/synthetic_image_generator.py | 16 +++++++++++++--- .../tests/test_synthetic_image_generator.py | 14 ++++++++++---- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py index 8dd0458e0..1238935b9 100644 --- a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py @@ -11,9 +11,19 @@ class ImageEncoding(Enum): BASE64 = auto() +class ImageFormat(Enum): + JPEG = auto() + PNG = auto() + + class SyntheticImageGenerator: - def __init__(self, image_encodeing: ImageEncoding): + def __init__( + self, + image_encodeing: ImageEncoding, + image_format: ImageFormat = ImageFormat.PNG, + ): self.image_encodeing = image_encodeing + self.image_format = image_format self._image_iterator = self.white_images_iterator() def __iter__(self): @@ -33,9 +43,9 @@ def _encode(self, image): return image elif self.image_encodeing == ImageEncoding.BASE64: buffered = BytesIO() - image.save(buffered, format="PNG") + image.save(buffered, format=self.image_format.name) data = base64.b64encode(buffered.getvalue()).decode("utf-8") - prefix = "data:image/png;base64" + prefix = f"data:image/{self.image_format.name.lower()};base64" return f"{prefix},{data}" else: raise GenAIPerfException( diff --git a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py index a7b594b51..3019b7ab4 100644 --- a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py @@ -4,6 +4,7 @@ import pytest from genai_perf.llm_inputs.synthetic_image_generator import ( ImageEncoding, + ImageFormat, SyntheticImageGenerator, ) from PIL import Image @@ -17,12 +18,15 @@ def test_generating_images(): assert isinstance(data, Image.Image), "generator produces unexpected type of data" -def test_base64_encoding(): - sut = SyntheticImageGenerator(image_encodeing=ImageEncoding.BASE64) +@pytest.mark.parametrize("image_format", [ImageFormat.PNG, ImageFormat.JPEG]) +def test_base64_encoding_with_different_formats(image_format): + sut = SyntheticImageGenerator( + image_encodeing=ImageEncoding.BASE64, image_format=image_format + ) base64String = next(sut) - base64prefix = "data:image/png;base64," + base64prefix = f"data:image/{image_format.name.lower()};base64," assert base64String.startswith(base64prefix), "unexpected prefix" data = base64String[len(base64prefix) :] @@ -30,4 +34,6 @@ def test_base64_encoding(): img_data = base64.b64decode(data) img_bytes = BytesIO(img_data) # test if an image is encoded - Image.open(img_bytes) + image = Image.open(img_bytes) + + assert image.format == image_format.name From 96307042361ed2385fc8412fe4204df212d8035f Mon Sep 17 00:00:00 2001 From: Marek Wawrzos Date: Thu, 11 Jul 2024 14:28:39 +0200 Subject: [PATCH 03/23] images should be base64-encoded arbitrarly --- .../llm_inputs/synthetic_image_generator.py | 35 +++++++------------ .../tests/test_synthetic_image_generator.py | 11 +++--- 2 files changed, 18 insertions(+), 28 deletions(-) diff --git a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py index 1238935b9..c8fd42546 100644 --- a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py @@ -6,23 +6,28 @@ from PIL import Image -class ImageEncoding(Enum): - PIL_IMAGE = auto() - BASE64 = auto() - - class ImageFormat(Enum): JPEG = auto() PNG = auto() +class Base64Encoder: + def __init__(self, image_format: ImageFormat = ImageFormat.PNG): + self.image_format = image_format + + def __call__(self, image): + buffered = BytesIO() + image.save(buffered, format=self.image_format.name) + data = base64.b64encode(buffered.getvalue()).decode("utf-8") + prefix = f"data:image/{self.image_format.name.lower()};base64" + return f"{prefix},{data}" + + class SyntheticImageGenerator: def __init__( self, - image_encodeing: ImageEncoding, image_format: ImageFormat = ImageFormat.PNG, ): - self.image_encodeing = image_encodeing self.image_format = image_format self._image_iterator = self.white_images_iterator() @@ -31,23 +36,9 @@ def __iter__(self): def __next__(self): image = next(self._image_iterator) - return self._encode(image) + return image def white_images_iterator(self): white_image = Image.new("RGB", (100, 100), color="white") while True: yield white_image - - def _encode(self, image): - if self.image_encodeing == ImageEncoding.PIL_IMAGE: - return image - elif self.image_encodeing == ImageEncoding.BASE64: - buffered = BytesIO() - image.save(buffered, format=self.image_format.name) - data = base64.b64encode(buffered.getvalue()).decode("utf-8") - prefix = f"data:image/{self.image_format.name.lower()};base64" - return f"{prefix},{data}" - else: - raise GenAIPerfException( - f"Unexpected image_encodeing: {self.image_encodeing.name}" - ) diff --git a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py index 3019b7ab4..8c44dd955 100644 --- a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py @@ -3,7 +3,7 @@ import pytest from genai_perf.llm_inputs.synthetic_image_generator import ( - ImageEncoding, + Base64Encoder, ImageFormat, SyntheticImageGenerator, ) @@ -11,7 +11,7 @@ def test_generating_images(): - sut = SyntheticImageGenerator(image_encodeing=ImageEncoding.PIL_IMAGE) + sut = SyntheticImageGenerator() data = next(sut) @@ -20,11 +20,10 @@ def test_generating_images(): @pytest.mark.parametrize("image_format", [ImageFormat.PNG, ImageFormat.JPEG]) def test_base64_encoding_with_different_formats(image_format): - sut = SyntheticImageGenerator( - image_encodeing=ImageEncoding.BASE64, image_format=image_format - ) + image = Image.new("RGB", (100, 100)) + sut = Base64Encoder(image_format=image_format) - base64String = next(sut) + base64String = sut(image) base64prefix = f"data:image/{image_format.name.lower()};base64," assert base64String.startswith(base64prefix), "unexpected prefix" From 7915dd7593e15b48dbcc8b0120fc787cb96e2660 Mon Sep 17 00:00:00 2001 From: Marek Wawrzos Date: Thu, 11 Jul 2024 15:48:04 +0200 Subject: [PATCH 04/23] randomized image format --- .../llm_inputs/synthetic_image_generator.py | 15 ++++++++------- .../tests/test_synthetic_image_generator.py | 4 ++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py index c8fd42546..e6d461d42 100644 --- a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py @@ -1,4 +1,5 @@ import base64 +import random from enum import Enum, auto from io import BytesIO @@ -11,24 +12,24 @@ class ImageFormat(Enum): PNG = auto() -class Base64Encoder: - def __init__(self, image_format: ImageFormat = ImageFormat.PNG): - self.image_format = image_format +class RandomFormatBase64Encoder: + def __init__(self, image_formats: ImageFormat = ImageFormat.PNG): + self.image_formats = image_formats def __call__(self, image): + choice = random.randint(0, len(self.image_formats) - 1) + image_format = self.image_formats[choice] buffered = BytesIO() - image.save(buffered, format=self.image_format.name) + image.save(buffered, format=image_format.name) data = base64.b64encode(buffered.getvalue()).decode("utf-8") - prefix = f"data:image/{self.image_format.name.lower()};base64" + prefix = f"data:image/{image_format.name.lower()};base64" return f"{prefix},{data}" class SyntheticImageGenerator: def __init__( self, - image_format: ImageFormat = ImageFormat.PNG, ): - self.image_format = image_format self._image_iterator = self.white_images_iterator() def __iter__(self): diff --git a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py index 8c44dd955..4ea4cb5b4 100644 --- a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py @@ -3,8 +3,8 @@ import pytest from genai_perf.llm_inputs.synthetic_image_generator import ( - Base64Encoder, ImageFormat, + RandomFormatBase64Encoder, SyntheticImageGenerator, ) from PIL import Image @@ -21,7 +21,7 @@ def test_generating_images(): @pytest.mark.parametrize("image_format", [ImageFormat.PNG, ImageFormat.JPEG]) def test_base64_encoding_with_different_formats(image_format): image = Image.new("RGB", (100, 100)) - sut = Base64Encoder(image_format=image_format) + sut = RandomFormatBase64Encoder(image_formats=[image_format]) base64String = sut(image) From 6176dc401340009f1d42ad5d4acf98484084a8c1 Mon Sep 17 00:00:00 2001 From: Marek Wawrzos Date: Thu, 11 Jul 2024 16:00:49 +0200 Subject: [PATCH 05/23] randomized image shape --- .../llm_inputs/synthetic_image_generator.py | 14 ++++++++++++-- .../tests/test_synthetic_image_generator.py | 16 ++++++++++++---- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py index e6d461d42..633f130cc 100644 --- a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py @@ -1,8 +1,8 @@ import base64 -import random from enum import Enum, auto from io import BytesIO +import numpy as np from genai_perf.exceptions import GenAIPerfException from PIL import Image @@ -17,7 +17,7 @@ def __init__(self, image_formats: ImageFormat = ImageFormat.PNG): self.image_formats = image_formats def __call__(self, image): - choice = random.randint(0, len(self.image_formats) - 1) + choice = np.random.randint(len(self.image_formats)) image_format = self.image_formats[choice] buffered = BytesIO() image.save(buffered, format=image_format.name) @@ -29,14 +29,24 @@ def __call__(self, image): class SyntheticImageGenerator: def __init__( self, + mean_size, + dimensions_stddev, ): self._image_iterator = self.white_images_iterator() + self.mean_size = mean_size + self.dimensions_stddev = dimensions_stddev def __iter__(self): return self + def random_resize(self, image): + new_size = np.random.normal(self.mean_size, self.dimensions_stddev) + new_size = new_size.astype(int) + return image.resize(new_size) + def __next__(self): image = next(self._image_iterator) + image = self.random_resize(image) return image def white_images_iterator(self): diff --git a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py index 4ea4cb5b4..75ec75e09 100644 --- a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py @@ -10,12 +10,20 @@ from PIL import Image -def test_generating_images(): - sut = SyntheticImageGenerator() +@pytest.mark.parametrize( + "image_size", + [ + (100, 100), + (200, 200), + ], +) +def test_different_image_size(image_size): + sut = SyntheticImageGenerator(mean_size=image_size, dimensions_stddev=[0, 0]) - data = next(sut) + image = next(sut) - assert isinstance(data, Image.Image), "generator produces unexpected type of data" + assert isinstance(image, Image.Image), "generator produces unexpected type of data" + assert image.size == image_size, "image not resized to the target size" @pytest.mark.parametrize("image_format", [ImageFormat.PNG, ImageFormat.JPEG]) From 9f8a42677ea684332d18f1d127944214ebb82b0e Mon Sep 17 00:00:00 2001 From: Marek Wawrzos Date: Thu, 11 Jul 2024 16:27:48 +0200 Subject: [PATCH 06/23] prepare SyntheticImageGenerator to support different image sources --- .../llm_inputs/synthetic_image_generator.py | 27 ++++++++++++++----- .../tests/test_synthetic_image_generator.py | 17 ++++++++++-- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py index 633f130cc..de8f427cb 100644 --- a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py @@ -1,6 +1,7 @@ import base64 from enum import Enum, auto from io import BytesIO +from typing import Tuple import numpy as np from genai_perf.exceptions import GenAIPerfException @@ -26,13 +27,30 @@ def __call__(self, image): return f"{prefix},{data}" +def white_images_generator(): + white_image = Image.new("RGB", (100, 100), color="white") + while True: + yield white_image + + +def build_synthetic_image_generator( + mean_size: Tuple[int, int], dimensions_stddev: Tuple[int, int] +): + return SyntheticImageGenerator( + mean_size=mean_size, + dimensions_stddev=dimensions_stddev, + image_iterator=white_images_generator(), + ) + + class SyntheticImageGenerator: def __init__( self, mean_size, dimensions_stddev, + image_iterator, ): - self._image_iterator = self.white_images_iterator() + self.image_iterator = image_iterator self.mean_size = mean_size self.dimensions_stddev = dimensions_stddev @@ -45,11 +63,6 @@ def random_resize(self, image): return image.resize(new_size) def __next__(self): - image = next(self._image_iterator) + image = next(self.image_iterator) image = self.random_resize(image) return image - - def white_images_iterator(self): - white_image = Image.new("RGB", (100, 100), color="white") - while True: - yield white_image diff --git a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py index 75ec75e09..0e0ae96a4 100644 --- a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py @@ -1,11 +1,13 @@ import base64 from io import BytesIO +import numpy as np import pytest from genai_perf.llm_inputs.synthetic_image_generator import ( ImageFormat, RandomFormatBase64Encoder, - SyntheticImageGenerator, + build_synthetic_image_generator, + white_images_generator, ) from PIL import Image @@ -18,7 +20,9 @@ ], ) def test_different_image_size(image_size): - sut = SyntheticImageGenerator(mean_size=image_size, dimensions_stddev=[0, 0]) + sut = build_synthetic_image_generator( + mean_size=image_size, dimensions_stddev=[0, 0] + ) image = next(sut) @@ -26,6 +30,15 @@ def test_different_image_size(image_size): assert image.size == image_size, "image not resized to the target size" +def test_white_images_generator(): + sut = white_images_generator() + + image = next(sut) + assert isinstance(image, Image.Image), "generator produces unexpected type of data" + white_pixel = np.array([[[255, 255, 255]]]) + assert (np.array(image) == white_pixel).all(), "not all pixels are white" + + @pytest.mark.parametrize("image_format", [ImageFormat.PNG, ImageFormat.JPEG]) def test_base64_encoding_with_different_formats(image_format): image = Image.new("RGB", (100, 100)) From 5178b277618394739b3520bf5203f5d6eae2834e Mon Sep 17 00:00:00 2001 From: Marek Wawrzos Date: Thu, 11 Jul 2024 22:50:04 +0200 Subject: [PATCH 07/23] read from files --- .../llm_inputs/synthetic_image_generator.py | 25 +++++++++-- .../tests/test_synthetic_image_generator.py | 44 +++++++++++++++++-- 2 files changed, 62 insertions(+), 7 deletions(-) diff --git a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py index de8f427cb..e12b3aafe 100644 --- a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py @@ -1,7 +1,9 @@ import base64 +import os from enum import Enum, auto from io import BytesIO -from typing import Tuple +from pathlib import Path +from typing import Optional, Tuple, cast import numpy as np from genai_perf.exceptions import GenAIPerfException @@ -27,6 +29,15 @@ def __call__(self, image): return f"{prefix},{data}" +def images_from_file_generator(image_path: Path): + if not image_path.exists(): + raise GenAIPerfException(f"File not found: {image_path}") + + image = Image.open(image_path) + while True: + yield image + + def white_images_generator(): white_image = Image.new("RGB", (100, 100), color="white") while True: @@ -34,12 +45,20 @@ def white_images_generator(): def build_synthetic_image_generator( - mean_size: Tuple[int, int], dimensions_stddev: Tuple[int, int] + mean_size: Tuple[int, int], + dimensions_stddev: Tuple[int, int], + image_path: Optional[Path] = None, ): + if image_path is None: + image_iterator = white_images_generator() + else: + image_path = cast(Path, image_path) + image_iterator = images_from_file_generator(image_path) + return SyntheticImageGenerator( mean_size=mean_size, dimensions_stddev=dimensions_stddev, - image_iterator=white_images_generator(), + image_iterator=image_iterator, ) diff --git a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py index 0e0ae96a4..5abe4246a 100644 --- a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py @@ -1,12 +1,16 @@ import base64 -from io import BytesIO +from io import BytesIO, StringIO +from pathlib import Path +from unittest.mock import patch import numpy as np import pytest +from genai_perf.exceptions import GenAIPerfException from genai_perf.llm_inputs.synthetic_image_generator import ( ImageFormat, RandomFormatBase64Encoder, - build_synthetic_image_generator, + SyntheticImageGenerator, + images_from_file_generator, white_images_generator, ) from PIL import Image @@ -20,8 +24,10 @@ ], ) def test_different_image_size(image_size): - sut = build_synthetic_image_generator( - mean_size=image_size, dimensions_stddev=[0, 0] + sut = SyntheticImageGenerator( + mean_size=image_size, + dimensions_stddev=[0, 0], + image_iterator=white_images_generator(), ) image = next(sut) @@ -30,6 +36,36 @@ def test_different_image_size(image_size): assert image.size == image_size, "image not resized to the target size" +@patch("pathlib.Path.exists", return_value=False) +def test_images_from_file_raises_when_file_not_found(mock_exists): + DUMMY_PATH = Path("dummy-image.png") + sut = images_from_file_generator(DUMMY_PATH) + + with pytest.raises(GenAIPerfException): + next(sut) + + +DUMMY_IMAGE = Image.new("RGB", (100, 100), color="blue") + + +@patch("pathlib.Path.exists", return_value=True) +@patch( + "PIL.Image.open", + return_value=DUMMY_IMAGE, +) +def test_images_from_file_generates_multiple_times(mock_file, mock_exists): + DUMMY_PATH = Path("dummy-image.png") + sut = images_from_file_generator(DUMMY_PATH) + + image = next(sut) + mock_exists.assert_called_once() + mock_file.assert_called_once_with(DUMMY_PATH) + assert image == DUMMY_IMAGE, "unexpected image produced" + + image = next(sut) + assert image == DUMMY_IMAGE, "unexpected image produced" + + def test_white_images_generator(): sut = white_images_generator() From 09e81a842c8e4895c40e68b1365c483761d7916c Mon Sep 17 00:00:00 2001 From: Marek Wawrzos Date: Fri, 12 Jul 2024 00:09:14 +0200 Subject: [PATCH 08/23] python 3.10 support fixes --- .../genai_perf/llm_inputs/synthetic_image_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py index e12b3aafe..078d2d451 100644 --- a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py @@ -78,7 +78,7 @@ def __iter__(self): def random_resize(self, image): new_size = np.random.normal(self.mean_size, self.dimensions_stddev) - new_size = new_size.astype(int) + new_size = tuple(new_size.astype(int)) return image.resize(new_size) def __next__(self): From 7f5d573a5e76d69a399b7212f8c4649432be4efd Mon Sep 17 00:00:00 2001 From: Marek Wawrzos Date: Fri, 12 Jul 2024 00:15:15 +0200 Subject: [PATCH 09/23] remove unused imports --- .../genai_perf/llm_inputs/synthetic_image_generator.py | 1 - .../genai-perf/tests/test_synthetic_image_generator.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py index 078d2d451..bc13490de 100644 --- a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py @@ -1,5 +1,4 @@ import base64 -import os from enum import Enum, auto from io import BytesIO from pathlib import Path diff --git a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py index 5abe4246a..a9918fca4 100644 --- a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py @@ -1,5 +1,5 @@ import base64 -from io import BytesIO, StringIO +from io import BytesIO from pathlib import Path from unittest.mock import patch From c4e7c352beecb21a5bf37fbf38599e000e054441 Mon Sep 17 00:00:00 2001 From: Marek Wawrzos Date: Fri, 12 Jul 2024 15:21:05 +0200 Subject: [PATCH 10/23] skip sampled image sizes with negative values --- .../llm_inputs/synthetic_image_generator.py | 13 +++++++++++-- .../tests/test_synthetic_image_generator.py | 11 +++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py index bc13490de..c3e76eb72 100644 --- a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py @@ -75,9 +75,18 @@ def __init__( def __iter__(self): return self + def _sample_random_positive_pair( + self, mean: Tuple[int, int], stddev: Tuple[int, int] + ) -> Tuple[int, int]: + new_size = np.array([-1, -1]) + while any(int(dim) <= 0 for dim in new_size): + new_size = np.random.normal(self.mean_size, self.dimensions_stddev) + return tuple(new_size.astype(int)) + def random_resize(self, image): - new_size = np.random.normal(self.mean_size, self.dimensions_stddev) - new_size = tuple(new_size.astype(int)) + new_size = self._sample_random_positive_pair( + self.mean_size, self.dimensions_stddev + ) return image.resize(new_size) def __next__(self): diff --git a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py index a9918fca4..acfba5635 100644 --- a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py @@ -36,6 +36,17 @@ def test_different_image_size(image_size): assert image.size == image_size, "image not resized to the target size" +def test_negative_size_is_not_selected(): + sut = SyntheticImageGenerator( + mean_size=(-1, -1), + dimensions_stddev=[10, 10], + image_iterator=white_images_generator(), + ) + + # exception is raised, when PIL.Image.resize is called with negative values + image = next(sut) + + @patch("pathlib.Path.exists", return_value=False) def test_images_from_file_raises_when_file_not_found(mock_exists): DUMMY_PATH = Path("dummy-image.png") From 5673a51fd9ad017307ad587e08e9c9e0fd16af8f Mon Sep 17 00:00:00 2001 From: Marek Wawrzos Date: Fri, 12 Jul 2024 15:25:00 +0200 Subject: [PATCH 11/23] formats type fix --- .../genai_perf/llm_inputs/synthetic_image_generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py index c3e76eb72..f8e778ed4 100644 --- a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py @@ -2,7 +2,7 @@ from enum import Enum, auto from io import BytesIO from pathlib import Path -from typing import Optional, Tuple, cast +from typing import List, Optional, Tuple, cast import numpy as np from genai_perf.exceptions import GenAIPerfException @@ -15,7 +15,7 @@ class ImageFormat(Enum): class RandomFormatBase64Encoder: - def __init__(self, image_formats: ImageFormat = ImageFormat.PNG): + def __init__(self, image_formats: List[ImageFormat] = [ImageFormat.PNG]): self.image_formats = image_formats def __call__(self, image): From d64cd27653c0bf4173a7fb1116c18dfb5d241e88 Mon Sep 17 00:00:00 2001 From: Marek Wawrzos Date: Fri, 12 Jul 2024 15:31:44 +0200 Subject: [PATCH 12/23] remove unused variable --- .../genai-perf/tests/test_synthetic_image_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py index acfba5635..6b9793da2 100644 --- a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py @@ -44,7 +44,7 @@ def test_negative_size_is_not_selected(): ) # exception is raised, when PIL.Image.resize is called with negative values - image = next(sut) + next(sut) @patch("pathlib.Path.exists", return_value=False) From 6d5b4ea537ba957632492b48d710a1b9da49dd32 Mon Sep 17 00:00:00 2001 From: Marek Wawrzos Date: Fri, 12 Jul 2024 16:26:54 +0200 Subject: [PATCH 13/23] synthetic image generator encodes images to base64 --- .../genai_perf/llm_inputs/synthetic_image_generator.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py index f8e778ed4..1e752cac6 100644 --- a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py @@ -47,6 +47,7 @@ def build_synthetic_image_generator( mean_size: Tuple[int, int], dimensions_stddev: Tuple[int, int], image_path: Optional[Path] = None, + formats: List[ImageFormat] = [ImageFormat.PNG], ): if image_path is None: image_iterator = white_images_generator() @@ -54,11 +55,13 @@ def build_synthetic_image_generator( image_path = cast(Path, image_path) image_iterator = images_from_file_generator(image_path) - return SyntheticImageGenerator( + image_generator = SyntheticImageGenerator( mean_size=mean_size, dimensions_stddev=dimensions_stddev, image_iterator=image_iterator, ) + base64_encode = RandomFormatBase64Encoder(formats) + return (base64_encode(image) for image in image_generator) class SyntheticImageGenerator: From edad485bcd36102080e88b96b8688c8b76e6c03f Mon Sep 17 00:00:00 2001 From: Marek Wawrzos Date: Mon, 15 Jul 2024 10:51:53 +0200 Subject: [PATCH 14/23] image format not randomized --- .../llm_inputs/synthetic_image_generator.py | 14 ++++++-------- .../tests/test_synthetic_image_generator.py | 2 +- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py index 1e752cac6..af0f8feef 100644 --- a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py @@ -15,16 +15,14 @@ class ImageFormat(Enum): class RandomFormatBase64Encoder: - def __init__(self, image_formats: List[ImageFormat] = [ImageFormat.PNG]): - self.image_formats = image_formats + def __init__(self, image_format: ImageFormat = ImageFormat.PNG): + self.image_format = image_format def __call__(self, image): - choice = np.random.randint(len(self.image_formats)) - image_format = self.image_formats[choice] buffered = BytesIO() - image.save(buffered, format=image_format.name) + image.save(buffered, format=self.image_format.name) data = base64.b64encode(buffered.getvalue()).decode("utf-8") - prefix = f"data:image/{image_format.name.lower()};base64" + prefix = f"data:image/{self.image_format.name.lower()};base64" return f"{prefix},{data}" @@ -47,7 +45,7 @@ def build_synthetic_image_generator( mean_size: Tuple[int, int], dimensions_stddev: Tuple[int, int], image_path: Optional[Path] = None, - formats: List[ImageFormat] = [ImageFormat.PNG], + image_format: ImageFormat = ImageFormat.PNG, ): if image_path is None: image_iterator = white_images_generator() @@ -60,7 +58,7 @@ def build_synthetic_image_generator( dimensions_stddev=dimensions_stddev, image_iterator=image_iterator, ) - base64_encode = RandomFormatBase64Encoder(formats) + base64_encode = RandomFormatBase64Encoder(image_format) return (base64_encode(image) for image in image_generator) diff --git a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py index 6b9793da2..36e40caec 100644 --- a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py @@ -89,7 +89,7 @@ def test_white_images_generator(): @pytest.mark.parametrize("image_format", [ImageFormat.PNG, ImageFormat.JPEG]) def test_base64_encoding_with_different_formats(image_format): image = Image.new("RGB", (100, 100)) - sut = RandomFormatBase64Encoder(image_formats=[image_format]) + sut = RandomFormatBase64Encoder(image_format=image_format) base64String = sut(image) From 935da0b86f0d08914f5833fe5fe5f41789437029 Mon Sep 17 00:00:00 2001 From: Marek Wawrzos Date: Mon, 15 Jul 2024 12:59:24 +0200 Subject: [PATCH 15/23] sample each dimension independently Co-authored-by: Hyunjae Woo <107147848+nv-hwoo@users.noreply.github.com> --- .../llm_inputs/synthetic_image_generator.py | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py index af0f8feef..6a566c131 100644 --- a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py @@ -76,19 +76,21 @@ def __init__( def __iter__(self): return self - def _sample_random_positive_pair( - self, mean: Tuple[int, int], stddev: Tuple[int, int] - ) -> Tuple[int, int]: - new_size = np.array([-1, -1]) - while any(int(dim) <= 0 for dim in new_size): - new_size = np.random.normal(self.mean_size, self.dimensions_stddev) - return tuple(new_size.astype(int)) + def _sample_random_positive_integer(self, mean: int, stddev: int) -> int: + while True: + n = int(np.random.normal(mean, stddev)) + if n > 0: + break + return n def random_resize(self, image): - new_size = self._sample_random_positive_pair( - self.mean_size, self.dimensions_stddev + width = self._sample_random_positive_integer( + self._image_width_mean, self._image_width_stddev ) - return image.resize(new_size) + height = self._sample_random_positive_integer( + self._image_height_mean, self._image_height_stddev + ) + return image.resize((width, height)) def __next__(self): image = next(self.image_iterator) From d8712eb232a74a7d518bb666600aad8e36a92963 Mon Sep 17 00:00:00 2001 From: Marek Wawrzos Date: Mon, 15 Jul 2024 13:48:32 +0200 Subject: [PATCH 16/23] apply code-review suggestsions --- .../genai_perf/llm_inputs/llm_inputs.py | 2 +- .../llm_inputs/synthetic_image_generator.py | 27 ++++++----- .../tests/test_synthetic_image_generator.py | 45 ++++++++++--------- 3 files changed, 41 insertions(+), 33 deletions(-) diff --git a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/llm_inputs.py b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/llm_inputs.py index 8f657ed42..838549c33 100644 --- a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/llm_inputs.py +++ b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/llm_inputs.py @@ -660,7 +660,7 @@ def _encode_images_in_input_dataset(cls, input_file_dataset: Dict) -> Dict: return input_file_dataset @classmethod - def _encode_image(cls, img: Image, format=ImageFormat.PNG): + def _encode_image(cls, img: Image.Image, format=ImageFormat.PNG): """Encodes an image into base64 encoding.""" buffered = BytesIO() img.save(buffered, format=format.name) diff --git a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py index 6a566c131..f68f5c75d 100644 --- a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py @@ -22,8 +22,7 @@ def __call__(self, image): buffered = BytesIO() image.save(buffered, format=self.image_format.name) data = base64.b64encode(buffered.getvalue()).decode("utf-8") - prefix = f"data:image/{self.image_format.name.lower()};base64" - return f"{prefix},{data}" + return f"data:image/{self.image_format.name.lower()};base64,{data}" def images_from_file_generator(image_path: Path): @@ -42,8 +41,10 @@ def white_images_generator(): def build_synthetic_image_generator( - mean_size: Tuple[int, int], - dimensions_stddev: Tuple[int, int], + image_width_mean: int, + image_height_mean: int, + image_width_stddev: int, + image_height_stddev: int, image_path: Optional[Path] = None, image_format: ImageFormat = ImageFormat.PNG, ): @@ -54,8 +55,10 @@ def build_synthetic_image_generator( image_iterator = images_from_file_generator(image_path) image_generator = SyntheticImageGenerator( - mean_size=mean_size, - dimensions_stddev=dimensions_stddev, + image_width_mean=image_width_mean, + image_height_mean=image_height_mean, + image_width_stddev=image_width_stddev, + image_height_stddev=image_height_stddev, image_iterator=image_iterator, ) base64_encode = RandomFormatBase64Encoder(image_format) @@ -65,13 +68,17 @@ def build_synthetic_image_generator( class SyntheticImageGenerator: def __init__( self, - mean_size, - dimensions_stddev, + image_width_mean, + image_height_mean, + image_width_stddev, + image_height_stddev, image_iterator, ): self.image_iterator = image_iterator - self.mean_size = mean_size - self.dimensions_stddev = dimensions_stddev + self._image_width_mean = image_width_mean + self._image_height_mean = image_height_mean + self._image_width_stddev = image_width_stddev + self._image_height_stddev = image_height_stddev def __iter__(self): return self diff --git a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py index 36e40caec..b966ca2da 100644 --- a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py @@ -17,29 +17,34 @@ @pytest.mark.parametrize( - "image_size", + "expected_image_size", [ (100, 100), (200, 200), ], ) -def test_different_image_size(image_size): +def test_different_image_size(expected_image_size): + expected_width, expected_height = expected_image_size sut = SyntheticImageGenerator( - mean_size=image_size, - dimensions_stddev=[0, 0], + image_width_mean=expected_width, + image_height_mean=expected_height, + image_width_stddev=0, + image_height_stddev=0, image_iterator=white_images_generator(), ) image = next(sut) assert isinstance(image, Image.Image), "generator produces unexpected type of data" - assert image.size == image_size, "image not resized to the target size" + assert image.size == expected_image_size, "image not resized to the target size" def test_negative_size_is_not_selected(): sut = SyntheticImageGenerator( - mean_size=(-1, -1), - dimensions_stddev=[10, 10], + image_width_mean=-1, + image_height_mean=-1, + image_width_stddev=10, + image_height_stddev=10, image_iterator=white_images_generator(), ) @@ -56,25 +61,21 @@ def test_images_from_file_raises_when_file_not_found(mock_exists): next(sut) -DUMMY_IMAGE = Image.new("RGB", (100, 100), color="blue") - - @patch("pathlib.Path.exists", return_value=True) -@patch( - "PIL.Image.open", - return_value=DUMMY_IMAGE, -) -def test_images_from_file_generates_multiple_times(mock_file, mock_exists): +def test_images_from_file_generates_multiple_times(mock_exists): + DUMMY_IMAGE = Image.new("RGB", (100, 100), color="blue") DUMMY_PATH = Path("dummy-image.png") - sut = images_from_file_generator(DUMMY_PATH) + with patch("PIL.Image.open", return_value=DUMMY_IMAGE) as mock_file: + sut = images_from_file_generator(DUMMY_PATH) - image = next(sut) - mock_exists.assert_called_once() - mock_file.assert_called_once_with(DUMMY_PATH) - assert image == DUMMY_IMAGE, "unexpected image produced" + image = next(sut) - image = next(sut) - assert image == DUMMY_IMAGE, "unexpected image produced" + mock_exists.assert_called_once() + mock_file.assert_called_once_with(DUMMY_PATH) + assert image == DUMMY_IMAGE, "unexpected image produced" + + image = next(sut) + assert image == DUMMY_IMAGE, "unexpected image produced" def test_white_images_generator(): From b5d4b64395eff22e4ce250e57d6852ae979f67bd Mon Sep 17 00:00:00 2001 From: Marek Wawrzos Date: Mon, 15 Jul 2024 11:21:23 +0200 Subject: [PATCH 17/23] update class name --- .../genai_perf/llm_inputs/synthetic_image_generator.py | 4 ++-- .../genai-perf/tests/test_synthetic_image_generator.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py index f68f5c75d..d9eeda634 100644 --- a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py @@ -14,7 +14,7 @@ class ImageFormat(Enum): PNG = auto() -class RandomFormatBase64Encoder: +class Base64Encoder: def __init__(self, image_format: ImageFormat = ImageFormat.PNG): self.image_format = image_format @@ -61,7 +61,7 @@ def build_synthetic_image_generator( image_height_stddev=image_height_stddev, image_iterator=image_iterator, ) - base64_encode = RandomFormatBase64Encoder(image_format) + base64_encode = Base64Encoder(image_format) return (base64_encode(image) for image in image_generator) diff --git a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py index b966ca2da..6b6bae7bd 100644 --- a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py @@ -7,8 +7,8 @@ import pytest from genai_perf.exceptions import GenAIPerfException from genai_perf.llm_inputs.synthetic_image_generator import ( + Base64Encoder, ImageFormat, - RandomFormatBase64Encoder, SyntheticImageGenerator, images_from_file_generator, white_images_generator, @@ -90,7 +90,7 @@ def test_white_images_generator(): @pytest.mark.parametrize("image_format", [ImageFormat.PNG, ImageFormat.JPEG]) def test_base64_encoding_with_different_formats(image_format): image = Image.new("RGB", (100, 100)) - sut = RandomFormatBase64Encoder(image_format=image_format) + sut = Base64Encoder(image_format=image_format) base64String = sut(image) From fb6e9829a60bdcb094ebd559a6d523ea3d6b31b0 Mon Sep 17 00:00:00 2001 From: Marek Wawrzos Date: Mon, 15 Jul 2024 11:34:56 +0200 Subject: [PATCH 18/23] deterministic synthetic image generator --- .../llm_inputs/synthetic_image_generator.py | 4 +++- .../tests/test_synthetic_image_generator.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py index d9eeda634..69af272f0 100644 --- a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py @@ -73,19 +73,21 @@ def __init__( image_width_stddev, image_height_stddev, image_iterator, + rng=None, ): self.image_iterator = image_iterator self._image_width_mean = image_width_mean self._image_height_mean = image_height_mean self._image_width_stddev = image_width_stddev self._image_height_stddev = image_height_stddev + self.rng = rng or np.random.default_rng() def __iter__(self): return self def _sample_random_positive_integer(self, mean: int, stddev: int) -> int: while True: - n = int(np.random.normal(mean, stddev)) + n = int(self.rng.normal(mean, stddev)) if n > 0: break return n diff --git a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py index 6b6bae7bd..302366701 100644 --- a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py @@ -52,6 +52,21 @@ def test_negative_size_is_not_selected(): next(sut) +def test_generator_deterministic(): + IMAGE_SIZE = 100, 100 + STDDEV = 100, 100 + SEED = 44 + img_gen1 = white_images_generator() + img_gen2 = white_images_generator() + rng1 = np.random.default_rng(seed=SEED) + rng2 = np.random.default_rng(seed=SEED) + sut1 = SyntheticImageGenerator(*IMAGE_SIZE, *STDDEV, img_gen1, rng=rng1) + sut2 = SyntheticImageGenerator(*IMAGE_SIZE, *STDDEV, img_gen2, rng=rng2) + + for _, img1, img2 in zip(range(5), sut1, sut2): + assert img1 == img2, "generator is nondererministic" + + @patch("pathlib.Path.exists", return_value=False) def test_images_from_file_raises_when_file_not_found(mock_exists): DUMMY_PATH = Path("dummy-image.png") From 287edbaaf6fe385fa604f3dac96919f9361eb8d4 Mon Sep 17 00:00:00 2001 From: Marek Wawrzos Date: Mon, 15 Jul 2024 12:52:31 +0200 Subject: [PATCH 19/23] add typing to SyntheticImageGenerator --- .../llm_inputs/synthetic_image_generator.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py index 69af272f0..960f9d64e 100644 --- a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py @@ -2,7 +2,7 @@ from enum import Enum, auto from io import BytesIO from pathlib import Path -from typing import List, Optional, Tuple, cast +from typing import Generator, List, Optional, Tuple, cast import numpy as np from genai_perf.exceptions import GenAIPerfException @@ -25,7 +25,7 @@ def __call__(self, image): return f"data:image/{self.image_format.name.lower()};base64,{data}" -def images_from_file_generator(image_path: Path): +def images_from_file_generator(image_path: Path) -> Generator[Image.Image, None, None]: if not image_path.exists(): raise GenAIPerfException(f"File not found: {image_path}") @@ -34,7 +34,7 @@ def images_from_file_generator(image_path: Path): yield image -def white_images_generator(): +def white_images_generator() -> Generator[Image.Image, None, None]: white_image = Image.new("RGB", (100, 100), color="white") while True: yield white_image @@ -68,12 +68,12 @@ def build_synthetic_image_generator( class SyntheticImageGenerator: def __init__( self, - image_width_mean, - image_height_mean, - image_width_stddev, - image_height_stddev, - image_iterator, - rng=None, + image_width_mean: int, + image_height_mean: int, + image_width_stddev: int, + image_height_stddev: int, + image_iterator: Generator[Image.Image, None, None], + rng: Optional[np.random.Generator] = None, ): self.image_iterator = image_iterator self._image_width_mean = image_width_mean From af0a93a88ba00c3844b0b0609560ab128ca43492 Mon Sep 17 00:00:00 2001 From: Marek Wawrzos Date: Mon, 15 Jul 2024 14:27:23 +0200 Subject: [PATCH 20/23] SyntheticImageGenerator doesn't load files --- .../llm_inputs/synthetic_image_generator.py | 49 ++----------------- .../tests/test_synthetic_image_generator.py | 45 +---------------- 2 files changed, 7 insertions(+), 87 deletions(-) diff --git a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py index 960f9d64e..653117b65 100644 --- a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py @@ -2,7 +2,7 @@ from enum import Enum, auto from io import BytesIO from pathlib import Path -from typing import Generator, List, Optional, Tuple, cast +from typing import Generator, Optional, Tuple, cast import numpy as np from genai_perf.exceptions import GenAIPerfException @@ -25,46 +25,6 @@ def __call__(self, image): return f"data:image/{self.image_format.name.lower()};base64,{data}" -def images_from_file_generator(image_path: Path) -> Generator[Image.Image, None, None]: - if not image_path.exists(): - raise GenAIPerfException(f"File not found: {image_path}") - - image = Image.open(image_path) - while True: - yield image - - -def white_images_generator() -> Generator[Image.Image, None, None]: - white_image = Image.new("RGB", (100, 100), color="white") - while True: - yield white_image - - -def build_synthetic_image_generator( - image_width_mean: int, - image_height_mean: int, - image_width_stddev: int, - image_height_stddev: int, - image_path: Optional[Path] = None, - image_format: ImageFormat = ImageFormat.PNG, -): - if image_path is None: - image_iterator = white_images_generator() - else: - image_path = cast(Path, image_path) - image_iterator = images_from_file_generator(image_path) - - image_generator = SyntheticImageGenerator( - image_width_mean=image_width_mean, - image_height_mean=image_height_mean, - image_width_stddev=image_width_stddev, - image_height_stddev=image_height_stddev, - image_iterator=image_iterator, - ) - base64_encode = Base64Encoder(image_format) - return (base64_encode(image) for image in image_generator) - - class SyntheticImageGenerator: def __init__( self, @@ -72,10 +32,8 @@ def __init__( image_height_mean: int, image_width_stddev: int, image_height_stddev: int, - image_iterator: Generator[Image.Image, None, None], rng: Optional[np.random.Generator] = None, ): - self.image_iterator = image_iterator self._image_width_mean = image_width_mean self._image_height_mean = image_height_mean self._image_width_stddev = image_width_stddev @@ -101,7 +59,10 @@ def random_resize(self, image): ) return image.resize((width, height)) + def get_next_image(self): + return Image.new("RGB", (100, 100), color="white") + def __next__(self): - image = next(self.image_iterator) + image = self.get_next_image() image = self.random_resize(image) return image diff --git a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py index 302366701..c218ef1de 100644 --- a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py @@ -10,8 +10,6 @@ Base64Encoder, ImageFormat, SyntheticImageGenerator, - images_from_file_generator, - white_images_generator, ) from PIL import Image @@ -30,7 +28,6 @@ def test_different_image_size(expected_image_size): image_height_mean=expected_height, image_width_stddev=0, image_height_stddev=0, - image_iterator=white_images_generator(), ) image = next(sut) @@ -45,7 +42,6 @@ def test_negative_size_is_not_selected(): image_height_mean=-1, image_width_stddev=10, image_height_stddev=10, - image_iterator=white_images_generator(), ) # exception is raised, when PIL.Image.resize is called with negative values @@ -56,52 +52,15 @@ def test_generator_deterministic(): IMAGE_SIZE = 100, 100 STDDEV = 100, 100 SEED = 44 - img_gen1 = white_images_generator() - img_gen2 = white_images_generator() rng1 = np.random.default_rng(seed=SEED) rng2 = np.random.default_rng(seed=SEED) - sut1 = SyntheticImageGenerator(*IMAGE_SIZE, *STDDEV, img_gen1, rng=rng1) - sut2 = SyntheticImageGenerator(*IMAGE_SIZE, *STDDEV, img_gen2, rng=rng2) + sut1 = SyntheticImageGenerator(*IMAGE_SIZE, *STDDEV, rng=rng1) + sut2 = SyntheticImageGenerator(*IMAGE_SIZE, *STDDEV, rng=rng2) for _, img1, img2 in zip(range(5), sut1, sut2): assert img1 == img2, "generator is nondererministic" -@patch("pathlib.Path.exists", return_value=False) -def test_images_from_file_raises_when_file_not_found(mock_exists): - DUMMY_PATH = Path("dummy-image.png") - sut = images_from_file_generator(DUMMY_PATH) - - with pytest.raises(GenAIPerfException): - next(sut) - - -@patch("pathlib.Path.exists", return_value=True) -def test_images_from_file_generates_multiple_times(mock_exists): - DUMMY_IMAGE = Image.new("RGB", (100, 100), color="blue") - DUMMY_PATH = Path("dummy-image.png") - with patch("PIL.Image.open", return_value=DUMMY_IMAGE) as mock_file: - sut = images_from_file_generator(DUMMY_PATH) - - image = next(sut) - - mock_exists.assert_called_once() - mock_file.assert_called_once_with(DUMMY_PATH) - assert image == DUMMY_IMAGE, "unexpected image produced" - - image = next(sut) - assert image == DUMMY_IMAGE, "unexpected image produced" - - -def test_white_images_generator(): - sut = white_images_generator() - - image = next(sut) - assert isinstance(image, Image.Image), "generator produces unexpected type of data" - white_pixel = np.array([[[255, 255, 255]]]) - assert (np.array(image) == white_pixel).all(), "not all pixels are white" - - @pytest.mark.parametrize("image_format", [ImageFormat.PNG, ImageFormat.JPEG]) def test_base64_encoding_with_different_formats(image_format): image = Image.new("RGB", (100, 100)) From e0b43fdcbc9dfaf1d2194c6ead22c32fcde82984 Mon Sep 17 00:00:00 2001 From: Marek Wawrzos Date: Mon, 15 Jul 2024 15:09:36 +0200 Subject: [PATCH 21/23] SyntheticImageGenerator always encodes images to base64 --- .../llm_inputs/synthetic_image_generator.py | 32 +++++++++---------- .../tests/test_synthetic_image_generator.py | 18 +++++++---- 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py index 653117b65..167578b95 100644 --- a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py @@ -14,17 +14,6 @@ class ImageFormat(Enum): PNG = auto() -class Base64Encoder: - def __init__(self, image_format: ImageFormat = ImageFormat.PNG): - self.image_format = image_format - - def __call__(self, image): - buffered = BytesIO() - image.save(buffered, format=self.image_format.name) - data = base64.b64encode(buffered.getvalue()).decode("utf-8") - return f"data:image/{self.image_format.name.lower()};base64,{data}" - - class SyntheticImageGenerator: def __init__( self, @@ -32,12 +21,14 @@ def __init__( image_height_mean: int, image_width_stddev: int, image_height_stddev: int, + image_format: ImageFormat = ImageFormat.PNG, rng: Optional[np.random.Generator] = None, ): self._image_width_mean = image_width_mean self._image_height_mean = image_height_mean self._image_width_stddev = image_width_stddev self._image_height_stddev = image_height_stddev + self.image_format = image_format self.rng = rng or np.random.default_rng() def __iter__(self): @@ -50,7 +41,7 @@ def _sample_random_positive_integer(self, mean: int, stddev: int) -> int: break return n - def random_resize(self, image): + def _random_resize(self, image): width = self._sample_random_positive_integer( self._image_width_mean, self._image_width_stddev ) @@ -59,10 +50,17 @@ def random_resize(self, image): ) return image.resize((width, height)) - def get_next_image(self): + def _get_next_image(self): return Image.new("RGB", (100, 100), color="white") - def __next__(self): - image = self.get_next_image() - image = self.random_resize(image) - return image + def _encode(self, image): + buffered = BytesIO() + image.save(buffered, format=self.image_format.name) + data = base64.b64encode(buffered.getvalue()).decode("utf-8") + return f"data:image/{self.image_format.name.lower()};base64,{data}" + + def __next__(self) -> str: + image = self._get_next_image() + image = self._random_resize(image) + base64_string = self._encode(image) + return base64_string diff --git a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py index c218ef1de..d61e52f21 100644 --- a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py @@ -7,13 +7,18 @@ import pytest from genai_perf.exceptions import GenAIPerfException from genai_perf.llm_inputs.synthetic_image_generator import ( - Base64Encoder, ImageFormat, SyntheticImageGenerator, ) from PIL import Image +def decode_image(base64_string): + _, data = base64_string.split(",") + decoded_data = base64.b64decode(data) + return Image.open(BytesIO(decoded_data)) + + @pytest.mark.parametrize( "expected_image_size", [ @@ -30,9 +35,9 @@ def test_different_image_size(expected_image_size): image_height_stddev=0, ) - image = next(sut) + base64_string = next(sut) + image = decode_image(base64_string) - assert isinstance(image, Image.Image), "generator produces unexpected type of data" assert image.size == expected_image_size, "image not resized to the target size" @@ -63,10 +68,11 @@ def test_generator_deterministic(): @pytest.mark.parametrize("image_format", [ImageFormat.PNG, ImageFormat.JPEG]) def test_base64_encoding_with_different_formats(image_format): - image = Image.new("RGB", (100, 100)) - sut = Base64Encoder(image_format=image_format) + IMAGE_SIZE = 100, 100 + STDDEV = 100, 100 + sut = SyntheticImageGenerator(*IMAGE_SIZE, *STDDEV, image_format=image_format) - base64String = sut(image) + base64String = next(sut) base64prefix = f"data:image/{image_format.name.lower()};base64," assert base64String.startswith(base64prefix), "unexpected prefix" From 1ef4f7193481004c44d5427d2592a67695cc40cb Mon Sep 17 00:00:00 2001 From: Marek Wawrzos Date: Mon, 15 Jul 2024 18:44:42 +0200 Subject: [PATCH 22/23] remove unused imports --- .../genai_perf/llm_inputs/synthetic_image_generator.py | 6 ++---- .../genai-perf/tests/test_synthetic_image_generator.py | 3 --- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py index 167578b95..a79832cf0 100644 --- a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py @@ -1,11 +1,9 @@ import base64 from enum import Enum, auto from io import BytesIO -from pathlib import Path -from typing import Generator, Optional, Tuple, cast +from typing import Optional, cast import numpy as np -from genai_perf.exceptions import GenAIPerfException from PIL import Image @@ -29,7 +27,7 @@ def __init__( self._image_width_stddev = image_width_stddev self._image_height_stddev = image_height_stddev self.image_format = image_format - self.rng = rng or np.random.default_rng() + self.rng = cast(np.random.Generator, rng or np.random.default_rng()) def __iter__(self): return self diff --git a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py index d61e52f21..16fb5b6f6 100644 --- a/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/tests/test_synthetic_image_generator.py @@ -1,11 +1,8 @@ import base64 from io import BytesIO -from pathlib import Path -from unittest.mock import patch import numpy as np import pytest -from genai_perf.exceptions import GenAIPerfException from genai_perf.llm_inputs.synthetic_image_generator import ( ImageFormat, SyntheticImageGenerator, From ae66dc3299d2dc4306176ac3ac85ddcda75b4ca6 Mon Sep 17 00:00:00 2001 From: Marek Wawrzos Date: Mon, 15 Jul 2024 20:31:23 +0200 Subject: [PATCH 23/23] generate gaussian noise instead of blank images --- .../genai_perf/llm_inputs/synthetic_image_generator.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py index a79832cf0..79aabd6da 100644 --- a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py +++ b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_inputs/synthetic_image_generator.py @@ -39,17 +39,16 @@ def _sample_random_positive_integer(self, mean: int, stddev: int) -> int: break return n - def _random_resize(self, image): + def _get_next_image(self): width = self._sample_random_positive_integer( self._image_width_mean, self._image_width_stddev ) height = self._sample_random_positive_integer( self._image_height_mean, self._image_height_stddev ) - return image.resize((width, height)) - - def _get_next_image(self): - return Image.new("RGB", (100, 100), color="white") + shape = width, height, 3 + noise = self.rng.integers(0, 256, shape, dtype=np.uint8) + return Image.fromarray(noise) def _encode(self, image): buffered = BytesIO() @@ -59,6 +58,5 @@ def _encode(self, image): def __next__(self) -> str: image = self._get_next_image() - image = self._random_resize(image) base64_string = self._encode(image) return base64_string