HistogramOptions now named to HistogramAndQuantilesOptions, implement num_quantiles option to HistogramAndQuantilesOptions

dataprofiler/profilers/__init__.py

@@ -28,7 +28,7 @@
     DataLabelerOptions,
     DateTimeOptions,
     FloatOptions,
-    HistogramOption,
+    HistogramAndQuantilesOption,
     HyperLogLogOptions,
     IntOptions,
     ModeOption,
@@ -66,7 +66,8 @@
 
 json_decoder._options = {
     BooleanOption.__name__: BooleanOption,
-    HistogramOption.__name__: HistogramOption,
+    "HistogramOption": HistogramAndQuantilesOption,
+    HistogramAndQuantilesOption.__name__: HistogramAndQuantilesOption,
     ModeOption.__name__: ModeOption,
     BaseInspectorOptions.__name__: BaseInspectorOptions,
     NumericalOptions.__name__: NumericalOptions,

dataprofiler/profilers/json_decoder.py

@@ -1,6 +1,7 @@
 """Contains methods to decode components of a Profiler."""
 from __future__ import annotations
 
+import warnings
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
@@ -72,6 +73,14 @@ def get_option_class(class_name: str) -> type[BaseOption]:
     options_class: type[BaseOption] | None = _options.get(class_name)
     if options_class is None:
         raise ValueError(f"Invalid option class {class_name} " f"failed to load.")
+
+    if class_name == "HistogramOption":
+        warnings.warn(
+            f"{class_name} will be deprecated in the future. During the JSON encode "
+            "process, HistogramOption is mapped to HistogramAndQuantilesOption. "
+            "Please begin utilizing the new HistogramAndQuantilesOption class.",
+            DeprecationWarning,
+        )
     return options_class
 
 

dataprofiler/profilers/numerical_column_stats.py

@@ -82,14 +82,15 @@ def __init__(self, options: NumericalOptions = None) -> None:
         self._mode_is_enabled: bool = True
         self.num_zeros: int | np.int64 = np.int64(0)
         self.num_negatives: int | np.int64 = np.int64(0)
-        self._num_quantiles: int = 1000  # TODO: add to options
+        self._num_quantiles: int = 1000  # By default, we use 1000 quantiles
 
         if options:
             self.bias_correction = options.bias_correction.is_enabled
             self._top_k_modes = options.mode.top_k_modes
             self._median_is_enabled = options.median.is_enabled
             self._median_abs_dev_is_enabled = options.median_abs_deviation.is_enabled
             self._mode_is_enabled = options.mode.is_enabled
+            self._num_quantiles = options.histogram_and_quantiles.num_quantiles
             bin_count_or_method = options.histogram_and_quantiles.bin_count_or_method
             if isinstance(bin_count_or_method, str):
                 self.histogram_bin_method_names = [bin_count_or_method]
@@ -113,7 +114,9 @@ def __init__(self, options: NumericalOptions = None) -> None:
                 "suggested_bin_count": self.min_histogram_bin,
                 "histogram": {"bin_counts": None, "bin_edges": None},
             }
-        self.quantiles: list[float] | None = None
+        self.quantiles: list[float] | dict = {
+            bin_num: None for bin_num in range(self._num_quantiles - 1)
+        }
         self.__calculations = {
             "min": NumericStatsMixin._get_min,
             "max": NumericStatsMixin._get_max,
@@ -459,6 +462,13 @@ def convert_histogram_key_types_to_np(histogram_info: dict):
                 self.histogram_methods[key]
             )
 
+        # Convert quantile keys to correct types
+        if isinstance(self.quantiles, dict):
+            new_quantiles = dict()
+            for key in self.quantiles.keys():
+                new_quantiles[int(key)] = self.quantiles[key]
+            self.quantiles = new_quantiles
+
         if self.min is not None:
             self.min = np.float64(self.min)
         if self.max is not None:

dataprofiler/profilers/profiler_options.py

@@ -210,13 +210,14 @@ def _validate_helper(self, variable_path: str = "BooleanOption") -> list[str]:
         return errors
 
 
-class HistogramOption(BooleanOption["HistogramOption"]):
+class HistogramAndQuantilesOption(BooleanOption["HistogramAndQuantilesOption"]):
     """For setting histogram options."""
 
     def __init__(
         self,
         is_enabled: bool = True,
         bin_count_or_method: str | int | list[str] = "auto",
+        num_quantiles: int = 1000,
     ) -> None:
         """
         Initialize Options for histograms.
@@ -226,11 +227,17 @@ def __init__(
         :ivar bin_count_or_method: bin count or the method with which to
             calculate histograms
         :vartype bin_count_or_method: Union[str, int, list(str)]
+        :ivar num_quantiles: boolean option to enable/disable num_quantiles
+            and set the number of quantiles
+        :vartype num_quantiles: int
         """
         self.bin_count_or_method = bin_count_or_method
+        self.num_quantiles = num_quantiles
         super().__init__(is_enabled=is_enabled)
 
-    def _validate_helper(self, variable_path: str = "HistogramOption") -> list[str]:
+    def _validate_helper(
+        self, variable_path: str = "HistogramAndQuantilesOption"
+    ) -> list[str]:
         """
         Validate the options do not conflict and cause errors.
 
@@ -260,6 +267,12 @@ def _validate_helper(self, variable_path: str = "HistogramOption") -> list[str]:
                     "than 1, a string, or list of strings from the "
                     "following: {}.".format(variable_path, valid_methods)
                 )
+
+        if self.num_quantiles is not None and (
+            not isinstance(self.num_quantiles, int) or self.num_quantiles < 1
+        ):
+            errors.append(f"{variable_path}.num_quantiles must be a positive integer.")
+
         return errors
 
 
@@ -396,7 +409,9 @@ def __init__(self) -> None:
         self.median_abs_deviation: BooleanOption = BooleanOption(is_enabled=True)
         self.num_zeros: BooleanOption = BooleanOption(is_enabled=True)
         self.num_negatives: BooleanOption = BooleanOption(is_enabled=True)
-        self.histogram_and_quantiles: HistogramOption = HistogramOption()
+        self.histogram_and_quantiles: HistogramAndQuantilesOption = (
+            HistogramAndQuantilesOption()
+        )
         # By default, we correct for bias
         self.bias_correction: BooleanOption = BooleanOption(is_enabled=True)
         BaseInspectorOptions.__init__(self)

dataprofiler/tests/profilers/profiler_options/test_float_options.py

@@ -94,7 +94,7 @@ def test_json_encode(self):
                     "data": {"is_enabled": True},
                 },
                 "histogram_and_quantiles": {
-                    "class": "HistogramOption",
+                    "class": "HistogramAndQuantilesOption",
                     "data": mock.ANY,
                 },
                 "bias_correction": {

dataprofiler/tests/profilers/profiler_options/test_histogram_option.py → dataprofiler/tests/profilers/profiler_options/test_histogram_and_quantiles_option.py

@@ -1,20 +1,23 @@
 import json
 
+from dataprofiler.profilers.json_decoder import load_option
 from dataprofiler.profilers.json_encoder import ProfileEncoder
-from dataprofiler.profilers.profiler_options import HistogramOption
+from dataprofiler.profilers.profiler_options import HistogramAndQuantilesOption
 
+from .. import utils as test_utils
 from .test_boolean_option import TestBooleanOption
 
 
-class TestHistogramOption(TestBooleanOption):
+class TestHistogramAndQuantilesOption(TestBooleanOption):
 
-    option_class = HistogramOption
+    option_class = HistogramAndQuantilesOption
     keys = []
 
     def test_init(self):
         option = self.get_options()
         self.assertTrue(option.is_enabled)
         self.assertEqual(option.bin_count_or_method, "auto")
+        self.assertEqual(option.num_quantiles, 1000)
 
     def test_set_helper(self):
         option = self.get_options()
@@ -26,6 +29,13 @@ def test_set_helper(self):
         with self.assertRaisesRegex(AttributeError, expected_error):
             option._set_helper({"bin_count_or_method.is_enabled": True}, "test")
 
+        # validate, variable path being passed
+        expected_error = (
+            "type object 'test.num_quantiles' has no attribute 'is_enabled'"
+        )
+        with self.assertRaisesRegex(AttributeError, expected_error):
+            option._set_helper({"num_quantiles.is_enabled": True}, "test")
+
     def test_set(self):
         option = self.get_options()
 
@@ -71,13 +81,58 @@ def _assert_set_helper(prop, value):
         with self.assertRaisesRegex(AttributeError, expected_error):
             option.set({"bin_count_or_method.is_enabled": True})
 
+        # Treat num_quantiles as a BooleanOption
+        expected_error = "type object 'num_quantiles' has no attribute " "'is_enabled'"
+        with self.assertRaisesRegex(AttributeError, expected_error):
+            option.set({"num_quantiles.is_enabled": True})
+
+        # Test set option for num_quantiles
+        option.set({"num_quantiles": 50})
+        self.assertEqual(option.num_quantiles, 50)
+
     def test_validate_helper(self):
         super().test_validate_helper()
 
+        optpth = self.get_options_path()
+
+        # Default configuration
+        option = self.get_options(num_quantiles=1000)
+        self.assertEqual([], option._validate_helper())
+
+        # Valid configurations
+        option = self.get_options(num_quantiles=50)
+        self.assertEqual([], option._validate_helper())
+        option = self.get_options(num_quantiles=2000)
+        self.assertEqual([], option._validate_helper())
+        option = self.get_options(num_quantiles=1)
+        self.assertEqual([], option._validate_helper())
+
+        # Option num_quantiles
+        option = self.get_options(num_quantiles="Hello World")
+        expected_error = [f"{optpth}.num_quantiles must be a positive integer."]
+        self.assertSetEqual(set(expected_error), set(option._validate_helper()))
+
+        # Option num_quantiles cannot be a float, must be an int
+        option = self.get_options(num_quantiles=1.1)
+        expected_error = [f"{optpth}.num_quantiles must be a positive integer."]
+        self.assertSetEqual(set(expected_error), set(option._validate_helper()))
+
+        # Option num_quantiles may not be zero, must be greater than one(1)
+        option = self.get_options(num_quantiles=0)
+        expected_error = [f"{optpth}.num_quantiles must be a positive integer."]
+        self.assertSetEqual(set(expected_error), set(option._validate_helper()))
+
+        # Option num_quantiles cannot be a negative integer
+        option = self.get_options(num_quantiles=-5)
+        expected_error = [f"{optpth}.num_quantiles must be a positive integer."]
+        self.assertSetEqual(set(expected_error), set(option._validate_helper()))
+
     def test_validate(self):
 
         super().test_validate()
 
+        optpth = self.get_options_path()
+
         params_to_check = [
             # non errors
             dict(prop="is_enabled", value_list=[False, True], errors=[]),
@@ -115,7 +170,7 @@ def test_validate(self):
                     "1",
                 ],
                 errors=[
-                    "HistogramOption.bin_count_or_method must be an integer "
+                    "HistogramAndQuantilesOption.bin_count_or_method must be an integer "
                     "more than 1, a string, or list of strings from the "
                     "following: ['auto', 'fd', 'doane', 'scott', 'rice', "
                     "'sturges', 'sqrt']."
@@ -155,13 +210,45 @@ def test_validate(self):
         # this time testing raising an error
         option.bin_count_or_method = "fake method"
         expected_error = (
-            r"HistogramOption.bin_count_or_method must be an integer more than "
+            r"HistogramAndQuantilesOption.bin_count_or_method must be an integer more than "
             r"1, a string, or list of strings from the following: "
             r"\['auto', 'fd', 'doane', 'scott', 'rice', 'sturges', 'sqrt']."
         )
         with self.assertRaisesRegex(ValueError, expected_error):
             option.validate()
 
+        # Valid configurations
+        option = self.get_options(num_quantiles=50)
+        self.assertEqual([], option._validate_helper())
+        option = self.get_options(num_quantiles=2000)
+        self.assertEqual([], option._validate_helper())
+        option = self.get_options(num_quantiles=1)
+        self.assertEqual([], option._validate_helper())
+
+        # Option num_quantiles cannot be a string, must be an int
+        option = self.get_options(num_quantiles="Hello World")
+        expected_error = f"{optpth}.num_quantiles must be a positive integer"
+        with self.assertRaisesRegex(ValueError, expected_error):
+            option.validate()
+
+        # Option num_quantiles cannot be a float, must be an int
+        option = self.get_options(num_quantiles=1.1)
+        expected_error = f"{optpth}.num_quantiles must be a positive integer"
+        with self.assertRaisesRegex(ValueError, expected_error):
+            option.validate()
+
+        # Option num_quantiles must be a positive integer
+        option = self.get_options(num_quantiles=0)
+        expected_error = f"{optpth}.num_quantiles must be a positive integer"
+        with self.assertRaisesRegex(ValueError, expected_error):
+            option.validate()
+
+        # Option num_quantiles cannot be a negative integer
+        option = self.get_options(num_quantiles=-5)
+        expected_error = f"{optpth}.num_quantiles must be a positive integer"
+        with self.assertRaisesRegex(ValueError, expected_error):
+            option.validate()
+
     def test_eq(self):
         super().test_eq()
 
@@ -173,18 +260,52 @@ def test_eq(self):
         self.assertNotEqual(options, options2)
         options2.bin_count_or_method = "sturges"
         self.assertEqual(options, options2)
+        options.num_quantiles = 30
+        self.assertNotEqual(options, options2)
+        options2.num_quantiles = 50
+        self.assertNotEqual(options, options2)
+        options2.num_quantiles = 30
+        self.assertEqual(options, options2)
 
     def test_json_encode(self):
-        option = HistogramOption(is_enabled=False, bin_count_or_method="doane")
+        option = HistogramAndQuantilesOption(
+            is_enabled=False, bin_count_or_method="doane"
+        )
 
         serialized = json.dumps(option, cls=ProfileEncoder)
 
         expected = {
-            "class": "HistogramOption",
+            "class": "HistogramAndQuantilesOption",
             "data": {
                 "bin_count_or_method": "doane",
+                "num_quantiles": 1000,
                 "is_enabled": False,
             },
         }
 
         self.assertDictEqual(expected, json.loads(serialized))
+
+    def test_json_decode_warn(self):
+        old_histogram = {
+            "class": "HistogramOption",
+            "data": {
+                "bin_count_or_method": "doane",
+                "is_enabled": False,
+            },
+        }
+
+        expected = HistogramAndQuantilesOption(
+            is_enabled=False, bin_count_or_method="doane"
+        )
+
+        expected_string = json.dumps(old_histogram, cls=ProfileEncoder)
+
+        expected_warning = (
+            "HistogramOption will be deprecated in the future. During the JSON encode "
+            "process, HistogramOption is mapped to HistogramAndQuantilesOption. "
+            "Please begin utilizing the new HistogramAndQuantilesOption class."
+        )
+
+        with self.assertWarnsRegex(DeprecationWarning, expected_warning):
+            deserialized = load_option(json.loads(expected_string))
+            test_utils.assert_profiles_equal(deserialized, expected)

dataprofiler/tests/profilers/profiler_options/test_int_options.py

@@ -86,7 +86,7 @@ def test_json_encode(self):
                     "data": {"is_enabled": True},
                 },
                 "histogram_and_quantiles": {
-                    "class": "HistogramOption",
+                    "class": "HistogramAndQuantilesOption",
                     "data": mock.ANY,
                 },
                 "bias_correction": {

dataprofiler/tests/profilers/profiler_options/test_numerical_options.py

@@ -422,7 +422,7 @@ def test_json_encode(self):
                     "data": {"is_enabled": True},
                 },
                 "histogram_and_quantiles": {
-                    "class": "HistogramOption",
+                    "class": "HistogramAndQuantilesOption",
                     "data": mock.ANY,
                 },
                 "bias_correction": {

dataprofiler/tests/profilers/profiler_options/test_profiler_options.py

@@ -95,7 +95,6 @@ def test_numerical_stats_option(self, *mocks):
                 self.assertIsNone(profile_column["statistics"]["min"])
                 self.assertIsNone(profile_column["statistics"]["max"])
                 self.assertTrue(np.isnan(profile_column["statistics"]["variance"]))
-                self.assertIsNone(profile_column["statistics"]["quantiles"])
                 self.assertTrue(np.isnan(profile_column["statistics"]["skewness"]))
                 self.assertTrue(np.isnan(profile_column["statistics"]["kurtosis"]))
 
@@ -242,7 +241,6 @@ def test_disabling_all_stats(self, *mocks):
                 self.assertIsNone(profile_column["statistics"]["min"])
                 self.assertIsNone(profile_column["statistics"]["max"])
                 self.assertTrue(np.isnan(profile_column["statistics"]["variance"]))
-                self.assertIsNone(profile_column["statistics"]["quantiles"])
                 self.assertTrue(profile_column["statistics"]["skewness"] is np.nan)
                 self.assertTrue(profile_column["statistics"]["kurtosis"] is np.nan)
                 self.assertTrue(