Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update aggregator.py #2995

Open
wants to merge 6 commits into
base: develop
Choose a base branch
from
+10 −3
Open

Update aggregator.py #2995

Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4a88a47
Update aggregator.py
zina-cs Sep 27, 2024
af21404
Update aggregator.py
zina-cs Oct 3, 2024
4c651d2
Update nncf/common/tensor_statistics/aggregator.py
zina-cs Oct 4, 2024
28adc78
Update nncf/common/tensor_statistics/aggregator.py
zina-cs Oct 4, 2024
12a4981
Update aggregator.py
zina-cs Oct 4, 2024
75fa16e
Update aggregator.py
zina-cs Oct 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 13 additions & 3 deletions nncf/common/tensor_statistics/aggregator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
from nncf.data.dataset import DataItem
from nncf.data.dataset import Dataset
from nncf.data.dataset import ModelInput
import warnings
zina-cs marked this conversation as resolved.
Show resolved Hide resolved


TensorType = TypeVar("TensorType")
TModel = TypeVar("TModel")
Expand All @@ -32,6 +34,8 @@
)




Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please avoid making unnecessary changes

class StatisticsAggregator(ABC):
"""
Base class for statistics collection.
Expand Down Expand Up @@ -70,7 +74,9 @@ def collect_statistics(self, model: TModel, graph: NNCFGraph) -> None:
engine = factory.EngineFactory.create(model_with_outputs)

iterations_number = self._get_iterations_number()
empty_statistics = True

processed_samples = 0

for input_data in track( # type: ignore
islice(self.dataset.get_inference_data(), iterations_number),
total=iterations_number,
Expand All @@ -79,9 +85,13 @@ def collect_statistics(self, model: TModel, graph: NNCFGraph) -> None:
outputs = engine.infer(input_data)
processed_outputs = self._process_outputs(outputs)
self._register_statistics(processed_outputs, merged_statistics)
empty_statistics = False
if empty_statistics:
processed_samples += 1

if processed_samples == 0:
raise nncf.ValidationError(EMPTY_DATASET_ERROR)

if subset_size > processed_samples:
warnings.warn(f"Dataset contains only {processed_samples} samples, smaller than the requested subset size {subset_size}.")
zina-cs marked this conversation as resolved.
Show resolved Hide resolved

def register_statistic_points(self, statistic_points: StatisticPointsContainer) -> None:
"""
Expand Down