Adding the 2nd set of functionality and many more widgets

orangecontrib/spark/utils/ml_api_utils.py

@@ -62,6 +62,12 @@ def get_models(self = None, module = None):
              'transform' in dir(c) and not inspect.isabstract(c) and 'java_model' not in inspect.getargspec(c).args and not name.startswith('Java') }
 
 
+def get_evaluators(self = None, module = None):
+    members = inspect.getmembers(module, inspect.isclass)
+    return { name: c for name, c in members if
+             'evaluate' in dir(c) and not inspect.isabstract(c) and 'java_model' not in inspect.getargspec(c).args and not name.startswith('Java') and name != 'Evaluator' }
+
+
 def get_transformers(self = None, module = None):
     members = inspect.getmembers(module, inspect.isclass)
     return { name: c for name, c in members if 'transform' in dir(c) and not inspect.isabstract(c) and not name.endswith('Model') and not name.startswith('Java') }

orangecontrib/spark/widgets/spark_ml_dataset.py

@@ -423,6 +423,8 @@ def set_data(self, data = None):
             self.available_attrs.extend(sorted(self.in_df.columns))
 
         else:
+            self.data = None
+            self.in_df = None
             self.used_attrs = VariablesListItemModel()
             self.class_attrs = VariablesListItemModel()
             self.meta_attrs = VariablesListItemModel()
@@ -574,7 +576,7 @@ def commit(self):
             VA = VectorAssembler(inputCols = attributes, outputCol = 'features')
             self.out_df = VA.transform(self.in_df)
             if len(class_var):
-                self.out_df = self.out_df.withColumn('label', self.out_df[class_var])
+                self.out_df = self.out_df.withColumn('label', self.out_df[class_var[0]])
 
             self.send("DataFrame", self.out_df)
         else:
@@ -583,10 +585,13 @@ def commit(self):
     def reset(self):
         if self.data is not None:
             self.available_attrs.extend(sorted(self.in_df.columns))
-            self.used_attrs = list()
-            self.class_attrs = list()
-            self.meta_attrs = list()
-            self.update_domain_role_hints()
+        else:
+            self.available_attrs = VariablesListItemModel()
+
+        self.used_attrs = VariablesListItemModel()
+        self.class_attrs = VariablesListItemModel()
+        self.meta_attrs = VariablesListItemModel()
+        self.update_domain_role_hints()
 
 
 def test_main(argv = None):

orangecontrib/spark/widgets/spark_ml_evaluation.py

@@ -0,0 +1,75 @@
+__author__ = 'jamh'
+
+import random
+
+from Orange.widgets import widget, gui
+from PyQt4 import QtGui, QtCore
+from pyspark.ml import evaluation
+
+from ..base.spark_ml_transformer import OWSparkTransformer
+from ..utils.ml_api_utils import get_evaluators
+
+
+class OWSparkMLEvaluator(OWSparkTransformer, widget.OWWidget):
+    name = "Evaluation"
+    description = "evaluation"
+    icon = "icons/spark.png"
+
+    module = evaluation
+    module_name = 'Evaluation'
+    box_text = "Spark Model Evaluator"
+    get_modules = get_evaluators
+
+    # outputs = [("DataFrame", pyspark.sql.DataFrame, widget.Dynamic)]
+
+    def __init__(self):
+        super().__init__()
+
+        # Create place to show/set parameters of method
+        self.values_box = gui.widgetBox(self.box, 'Evaluation:', addSpace = True)
+
+        self.table = QtGui.QTableWidget(self.values_box)
+        self.tableItem = QtGui.QTableWidgetItem()
+
+        self.values_box.hide()
+
+    def refresh_method(self, text):
+        super().refresh_method(text)
+        if hasattr(self, 'values_box'):
+            self.values_box.hide()
+
+    def transform(self):
+        metric_names = self.gui_parameters['metricName'].doc_text.split('(')[-1].replace(')', '').split('|')
+        values = { }
+        print(metric_names)
+        if self.in_df:
+            for metric in metric_names:
+                values[metric] = self.method.transform(self.in_df)
+        else:
+            for k in metric_names:
+                values[k] = round(5 * random.random() - 2.5, 2)
+
+        print(values.items())
+
+        # self.send("DataFrame", self.out_df)
+        self.table.clear()
+        self.table.resize(500, 500)
+        self.table.setRowCount(len(values))
+        self.table.setColumnCount(2)
+
+        # set label
+        self.table.setHorizontalHeaderLabels(["Metric", "Value"])
+        # self.table.setVerticalHeaderLabels(list(values))
+
+        # set data
+        for i, kv in enumerate(values.items()):
+            k, v = kv
+            it1 = QtGui.QTableWidgetItem(k)
+            it2 = QtGui.QTableWidgetItem(str(v))
+            self.table.setItem(i, 0, it1)
+            self.table.setItem(i, 1, it2)
+            it1.setFlags(QtCore.Qt.ItemIsEnabled)
+            it2.setFlags(QtCore.Qt.ItemIsEnabled)
+
+        # show table
+        self.values_box.show()