diff --git a/ckanext/charts/chart_builders/base.py b/ckanext/charts/chart_builders/base.py index cb61d1c..f9ed732 100644 --- a/ckanext/charts/chart_builders/base.py +++ b/ckanext/charts/chart_builders/base.py @@ -302,7 +302,6 @@ def chart_xlabel_field(self) -> dict[str, Any]: "group": "Styles", "validators": [ self.get_validator("ignore_empty"), - # self.get_validator("default")(" "), self.get_validator("unicode_safe"), ], } @@ -315,7 +314,6 @@ def chart_ylabel_field(self) -> dict[str, Any]: "group": "Styles", "validators": [ self.get_validator("ignore_empty"), - # self.get_validator("default")(" "), self.get_validator("unicode_safe"), ], } @@ -362,7 +360,7 @@ def type_field(self, choices: list[dict[str, str]]) -> dict[str, Any]: "choices": choices, "group": "Structure", "validators": [ - self.get_validator("default")("Bar"), + self.get_validator("default")("Line"), self.get_validator("unicode_safe"), ], "form_attrs": { @@ -485,6 +483,19 @@ def split_data_field(self) -> dict[str, Any]: on datetime column stated for the x-axis""" } + def skip_null_values_field(self) -> dict[str, Any]: + return { + "field_name": "skip_null_values", + "label": "Skip N/A and NULL values", + "form_snippet": "chart_checkbox.html", + "group": "Data", + "validators": [ + self.get_validator("boolean_validator"), + ], + "help_text": """Entries in the data with N/A or NULL will not be + graphed and will be skipped""" + } + def sort_x_field(self) -> dict[str, Any]: return { "field_name": "sort_x", diff --git a/ckanext/charts/chart_builders/plotly.py b/ckanext/charts/chart_builders/plotly.py index 532af75..00de64b 100644 --- a/ckanext/charts/chart_builders/plotly.py +++ b/ckanext/charts/chart_builders/plotly.py @@ -46,7 +46,7 @@ class PlotlyLineBuilder(PlotlyBuilder): def to_json(self) -> Any: return self.build_line_chart() - def split_data_by_year(self) -> dict[str, Any]: + def _split_data_by_year(self) -> dict[str, Any]: """ Prepare data for a line chart. It splits the data by year stated in the date format column which is used for x-axis. @@ -63,20 +63,31 @@ def split_data_by_year(self) -> dict[str, Any]: return self + def _skip_null_values(self, column) -> tuple[Any]: + if self.settings.get("skip_null_values", True): + x = self.df[self.df[column].notna()][self.settings["x"]] + y = self.df[self.df[column].notna()][column] + else: + x = self.df[self.settings["x"]] + y = self.df[column].fillna(0) + return x, y + def build_line_chart(self) -> Any: """ Build a line chart. It supports multi columns for y-axis to display on the line chart. """ if self.settings.get("split_data", False): - self.split_data_by_year() + self._split_data_by_year() + + x, y = self._skip_null_values(self.settings["y"][0]) fig = make_subplots(specs=[[{"secondary_y": True}]]) fig.add_trace( go.Scatter( - x=self.df[self.settings["x"]], - y=self.df[self.settings["y"][0]], + x=x, + y=y, name=self.settings["y"][0], ), secondary_y=False, @@ -84,10 +95,12 @@ def build_line_chart(self) -> Any: if len(self.settings["y"]) > 1: for column in self.settings["y"][1:]: + x, y = self._skip_null_values(column) + fig.add_trace( go.Scatter( - x=self.df[self.settings["x"]], - y=self.df[column], + x=x, + y=y, name=column, ), secondary_y=True, @@ -230,6 +243,7 @@ def get_form_fields(self): self.sort_x_field(), self.sort_y_field(), self.split_data_field(), + self.skip_null_values_field(), self.limit_field(maximum=1000000), self.chart_title_field(), self.chart_xlabel_field(), diff --git a/ckanext/charts/fetchers.py b/ckanext/charts/fetchers.py index 5ea8ba0..25c6838 100644 --- a/ckanext/charts/fetchers.py +++ b/ckanext/charts/fetchers.py @@ -83,7 +83,7 @@ def fetch_data(self) -> pd.DataFrame: log.warning(f"Warning: Could not convert date_time column: {e}") # Apply numeric conversion to all columns - it will safely ignore non-numeric values - df = df.apply(pd.to_numeric, errors='ignore').fillna(0) + df = df.apply(pd.to_numeric, errors='ignore') except (ProgrammingError, UndefinedTable) as e: raise exception.DataFetchError( diff --git a/ckanext/charts/templates/scheming/form_snippets/chart_checkbox.html b/ckanext/charts/templates/scheming/form_snippets/chart_checkbox.html index 796afb6..af83fa2 100644 --- a/ckanext/charts/templates/scheming/form_snippets/chart_checkbox.html +++ b/ckanext/charts/templates/scheming/form_snippets/chart_checkbox.html @@ -1,13 +1,14 @@ {% set val = data[field.field_name] %} +{% set skip_null = field.field_name=="skip_null_values" %}