Merge pull request #18 from DataShades/SXLLCAXAAD-3

SXLLCAXAAD-3 / skip NaN values in chart

ckanext/charts/chart_builders/base.py

@@ -302,7 +302,6 @@ def chart_xlabel_field(self) -> dict[str, Any]:
             "group": "Styles",
             "validators": [
                 self.get_validator("ignore_empty"),
-                # self.get_validator("default")(" "),
                 self.get_validator("unicode_safe"),
             ],
         }
@@ -315,7 +314,6 @@ def chart_ylabel_field(self) -> dict[str, Any]:
             "group": "Styles",
             "validators": [
                 self.get_validator("ignore_empty"),
-                # self.get_validator("default")(" "),
                 self.get_validator("unicode_safe"),
             ],
         }
@@ -362,7 +360,7 @@ def type_field(self, choices: list[dict[str, str]]) -> dict[str, Any]:
             "choices": choices,
             "group": "Structure",
             "validators": [
-                self.get_validator("default")("Bar"),
+                self.get_validator("default")("Line"),
                 self.get_validator("unicode_safe"),
             ],
             "form_attrs": {
@@ -485,6 +483,19 @@ def split_data_field(self) -> dict[str, Any]:
                             on datetime column stated for the x-axis"""
         }
 
+    def skip_null_values_field(self) -> dict[str, Any]:
+        return {
+            "field_name": "skip_null_values",
+            "label": "Skip N/A and NULL values",
+            "form_snippet": "chart_checkbox.html",
+            "group": "Data",
+            "validators": [
+                self.get_validator("boolean_validator"),
+            ],
+            "help_text":    """Entries in the data with N/A or NULL will not be 
+                            graphed and will be skipped"""
+        }
+
     def sort_x_field(self) -> dict[str, Any]:
         return {
             "field_name": "sort_x",

ckanext/charts/chart_builders/plotly.py

@@ -46,7 +46,7 @@ class PlotlyLineBuilder(PlotlyBuilder):
     def to_json(self) -> Any:
         return self.build_line_chart()
 
-    def split_data_by_year(self) -> dict[str, Any]:
+    def _split_data_by_year(self) -> dict[str, Any]:
         """
         Prepare data for a line chart. It splits the data by year stated
         in the date format column which is used for x-axis.
@@ -63,31 +63,44 @@ def split_data_by_year(self) -> dict[str, Any]:
 
         return self
 
+    def _skip_null_values(self, column) -> tuple[Any]:
+        if self.settings.get("skip_null_values", True):
+            x = self.df[self.df[column].notna()][self.settings["x"]]
+            y = self.df[self.df[column].notna()][column]
+        else:
+            x = self.df[self.settings["x"]]
+            y = self.df[column].fillna(0)
+        return x, y
+
     def build_line_chart(self) -> Any:
         """
         Build a line chart. It supports multi columns for y-axis
         to display on the line chart.
         """
         if self.settings.get("split_data", False):
-            self.split_data_by_year()
+            self._split_data_by_year()
+
+        x, y = self._skip_null_values(self.settings["y"][0])
 
         fig = make_subplots(specs=[[{"secondary_y": True}]])
 
         fig.add_trace(
             go.Scatter(
-                x=self.df[self.settings["x"]],
-                y=self.df[self.settings["y"][0]],
+                x=x,
+                y=y,
                 name=self.settings["y"][0],
             ),
             secondary_y=False,
         )
 
         if len(self.settings["y"]) > 1:
             for column in self.settings["y"][1:]:
+                x, y = self._skip_null_values(column)
+
                 fig.add_trace(
                     go.Scatter(
-                        x=self.df[self.settings["x"]],
-                        y=self.df[column],
+                        x=x,
+                        y=y,
                         name=column,
                     ),
                     secondary_y=True,
@@ -230,6 +243,7 @@ def get_form_fields(self):
             self.sort_x_field(),
             self.sort_y_field(),
             self.split_data_field(),
+            self.skip_null_values_field(),
             self.limit_field(maximum=1000000),
             self.chart_title_field(),
             self.chart_xlabel_field(),

ckanext/charts/fetchers.py

@@ -83,7 +83,7 @@ def fetch_data(self) -> pd.DataFrame:
                     log.warning(f"Warning: Could not convert date_time column: {e}")
 
             # Apply numeric conversion to all columns - it will safely ignore non-numeric values
-            df = df.apply(pd.to_numeric, errors='ignore').fillna(0)
+            df = df.apply(pd.to_numeric, errors='ignore')
 
         except (ProgrammingError, UndefinedTable) as e:
             raise exception.DataFetchError(

ckanext/charts/templates/scheming/form_snippets/chart_checkbox.html

@@ -1,13 +1,14 @@
 {% set val = data[field.field_name] %}
+{% set skip_null = field.field_name=="skip_null_values" %}
 
 <div class="form-group">
 	<label for="field-{{ field.field_name }}" class="no-dots">
-	    <input id="field-{{ field.field_name }}"
-	        type="checkbox"
-	        name="{{ field.field_name }}"
-	        value="true"
-	        {{"checked " if (val is string and 'true' in val) or val }} />
-	    {{ field.label }}
+		<input id="field-{{ field.field_name }}"
+			type="checkbox"
+			name="{{ field.field_name }}"
+			value="true"
+			{{"checked " if (skip_null and not val) or (val is string and 'true' in val) or val }} />
+		{{ field.label }}
 	</label>
 
 	{%- snippet 'scheming/form_snippets/help_text.html', field=field -%}