Adding the 2nd set of functionality and many more widgets

orangecontrib/spark/widgets/__init__.py

@@ -1,20 +1,6 @@
-# Category metadata.
-
-# Category icon show in the menu
-ICON = "icons/category.ico"
-
-# Background color for category background in menu
-# and widget icon background in workflow.
-#BACKGROUND = "light-orange"
-BACKGROUND = "white"
-
-# Location of widget help files.
-WIDGET_HELP_PATH = (
-    # Used for development.
-    # You still need to build help pages using
-    # make htmlhelp
-    # inside doc folder
-    ("{DEVELOP_ROOT}/doc/build/htmlhelp/index.html", None),
-
-    ("http://spark.com/my-widget/docs/", "")
-)
+__author__ = "Jose Antonio Martin H."
+__copyright__ = "Copyright 2015, Jose Antonio Martin H."
+__credits__ = ["The Orange Machine Learning Project, Jose Antonio Martin H. "]
+__license__ = "Apache License 2.0"
+__maintainer__ = "JOse Antonio Martin H."
+__email__ = "[email protected]"

orangecontrib/spark/widgets/data/__init__.py

@@ -0,0 +1,29 @@
+__author__ = "Jose Antonio Martin H."
+__copyright__ = "Copyright 2015, Jose Antonio Martin H."
+__credits__ = ["The Orange Machine Learning Project, Jose Antonio Martin H. "]
+__license__ = "Apache License 2.0"
+__maintainer__ = "JOse Antonio Martin H."
+__email__ = "[email protected]"
+
+
+# Category metadata.
+
+# Category icon show in the menu
+ICON = "../icons/category.ico"
+
+# Background color for category background in menu
+# and widget icon background in workflow.
+#BACKGROUND = "light-orange"
+BACKGROUND = "white"
+
+# Location of widget help files.
+WIDGET_HELP_PATH = (
+    # Used for development.
+    # You still need to build help pages using
+    # make htmlhelp
+    # inside doc folder
+    ("{DEVELOP_ROOT}/doc/build/htmlhelp/index.html", None),
+
+    ("http://spark.com/my-widget/docs/", "")
+)
+

orangecontrib/spark/widgets/odbc_table.py → orangecontrib/spark/widgets/data/odbc_table.py

@@ -31,7 +31,7 @@ class OWodbcTable(OWWidget):
     settingsList = ["recentConnections", "lastQuery"]
     name = "ODBC"
     description = "Create a Table from an ODBC datasource"
-    icon = "icons/sql.png"
+    icon = "../icons/sql.png"
     inputs = []
     outputs = [("Data", Orange.data.Table, widget.Default),
                ("Feature Definitions", Orange.data.Domain, widget.Default),

orangecontrib/spark/widgets/data/orange_to_pandas.py

@@ -0,0 +1,24 @@
+__author__ = 'jamh'
+
+import pandas
+from Orange.data import Table
+from Orange.widgets import widget, gui, settings
+
+from orangecontrib.spark.utils.bdutils import orange_to_pandas
+
+
+class OWOrangeToPandas(widget.OWWidget):
+    name = "to Pandas"
+    description = "Convert Orange Table to Pandas DataFrame"
+    icon = "../icons/orange.ico"
+
+    inputs = [("Table", Table, "get_input", widget.Default)]
+    outputs = [("Pandas", pandas.DataFrame, widget.Dynamic)]
+    settingsHandler = settings.DomainContextHandler()
+
+    def __init__(self):
+        super().__init__()
+        gui.label(self.controlArea, self, "to pandas:")
+
+    def get_input(self, obj):
+        self.send("Pandas", orange_to_pandas(obj))

orangecontrib/spark/widgets/pandas_to_orange.py → orangecontrib/spark/widgets/data/pandas_to_orange.py

@@ -3,13 +3,13 @@
 import pandas
 from Orange.widgets import widget, gui, settings
 
-from ..utils.bdutils import pandas_to_orange
+from orangecontrib.spark.utils.bdutils import pandas_to_orange
 
 
 class OWPandasToOrange(widget.OWWidget):
-    name = "from pandas"
+    name = "from Pandas"
     description = "Convert Pandas DataFrame to Orange Table"
-    icon = "icons/orange.ico"
+    icon = "../icons/orange.ico"
     inputs = [("Pandas", pandas.DataFrame, "get_input", widget.Default)]
     outputs = [("Table", Table, widget.Dynamic)]
     settingsHandler = settings.DomainContextHandler()

orangecontrib/spark/widgets/pyspark_script_console.py → orangecontrib/spark/widgets/data/pyspark_script_console.py

@@ -354,13 +354,13 @@ def select_row(view, row):
                     QItemSelectionModel.ClearAndSelect)
 
 
-from ..base.shared_spark_context import SharedSparkContext
+from orangecontrib.spark.base.shared_spark_context import SharedSparkContext
 
 
 class OWPySparkScript(SharedSparkContext, widget.OWWidget):
     name = "PySpark Script"
     description = "Write a PySpark script and run it on input"
-    icon = "icons/PythonScript.svg"
+    icon = "../icons/PythonScript.svg"
     priority = 3150
 
     inputs = [("in_object", object, "setObject")]

orangecontrib/spark/widgets/spark_context.py → orangecontrib/spark/widgets/data/spark_context.py

@@ -5,15 +5,15 @@
 from pyspark import SparkConf, SparkContext
 from pyspark.sql import HiveContext
 
-from ..base.shared_spark_context import SharedSparkContext
-from ..utils.gui_utils import GuiParam
+from orangecontrib.spark.base.shared_spark_context import SharedSparkContext
+from orangecontrib.spark.utils.gui_utils import GuiParam
 
 
 class OWSparkContext(SharedSparkContext, widget.OWWidget):
     priority = 0
     name = "Context"
     description = "Create a shared Spark (sc) and Hive (hc) Contexts"
-    icon = "icons/spark.png"
+    icon = "../icons/spark.png"
 
     want_main_area = False
     resizing_enabled = True

orangecontrib/spark/widgets/data/spark_from_orange.py

@@ -0,0 +1,27 @@
+__author__ = 'jamh'
+
+import pyspark
+from Orange.data import Table
+from Orange.widgets import widget, gui, settings
+
+from orangecontrib.spark.base.shared_spark_context import SharedSparkContext
+from orangecontrib.spark.utils.bdutils import orange_to_pandas
+
+
+class OWSparkFromOrange(SharedSparkContext, widget.OWWidget):
+    name = "from Orange"
+    description = "Convert Orange Table to Spark DataFrame"
+    icon = "../icons/spark.ico"
+
+    inputs = [("Table", Table, "get_input", widget.Default)]
+    outputs = [("DataFrame", pyspark.sql.DataFrame, widget.Dynamic)]
+    settingsHandler = settings.DomainContextHandler()
+
+    auto_commit = settings.Setting(True)
+
+    def __init__(self):
+        super().__init__()
+        gui.label(self.controlArea, self, "From Oranges:")
+
+    def get_input(self, obj):
+        self.send("DataFrame", self.hc.createDataFrame(orange_to_pandas(obj)))

orangecontrib/spark/widgets/data/spark_from_pandas.py

@@ -0,0 +1,26 @@
+__author__ = 'jamh'
+
+import pandas
+import pyspark
+from Orange.widgets import widget, gui, settings
+from PyQt4.QtGui import QSizePolicy
+
+from orangecontrib.spark.base.shared_spark_context import SharedSparkContext
+
+
+class OWSparkToPandas(SharedSparkContext, widget.OWWidget):
+    name = "from Pandas"
+    description = "Convert Pandas dataframe to Spark DataFrame."
+    icon = "../icons/spark.ico"
+
+    inputs = [("DataFrame", pandas.DataFrame, "get_input", widget.Default)]
+    outputs = [("DataFrame", pyspark.sql.DataFrame, widget.Dynamic)]
+    settingsHandler = settings.DomainContextHandler()
+
+    def __init__(self):
+        super().__init__()
+        gui.label(self.controlArea, self, "Pandas->Spark:")
+        self.setSizePolicy(QSizePolicy.Maximum, QSizePolicy.Maximum)
+
+    def get_input(self, obj):
+        self.send("DataFrame", self.hc.createDataFrame(obj))

orangecontrib/spark/widgets/spark_SQL_dataframe.py → orangecontrib/spark/widgets/data/spark_sql_dataframe.py

@@ -7,7 +7,7 @@
 )
 
 from orangecontrib.spark.utils.bdutils import pandas_to_orange, format_sql
-from ..base.shared_spark_context import SharedSparkContext
+from orangecontrib.spark.base.shared_spark_context import SharedSparkContext
 
 
 def convert_dataframe_to_orange(df):
@@ -19,7 +19,7 @@ class OWSparkDataFrame(SharedSparkContext, OWWidget):
     settingsList = ["lastQuery"]
     name = "DataFrame"
     description = "Create a Spark Dataframe from an SparkSQL source"
-    icon = "icons/sql.png"
+    icon = "../icons/sql.png"
 
     outputs = [("DataFrame", pyspark.sql.DataFrame, widget.Dynamic)]
     out_df = None

orangecontrib/spark/widgets/spark_table.py → orangecontrib/spark/widgets/data/spark_table.py

@@ -5,14 +5,14 @@
 from Orange.widgets import widget, gui
 from pyspark.sql import HiveContext
 
-from ..base.shared_spark_context import SharedSparkContext
-from ..utils.gui_utils import GuiParam
+from orangecontrib.spark.base.shared_spark_context import SharedSparkContext
+from orangecontrib.spark.utils.gui_utils import GuiParam
 
 
 class OWSparkSQLTableContext(SharedSparkContext, widget.OWWidget):
     name = "Hive Table"
     description = "Create a Spark DataFrame from a Hive Table"
-    icon = "icons/Table.svg"
+    icon = "../icons/Hive.png"
     outputs = [("DataFrame", pyspark.sql.DataFrame, widget.Dynamic)]
 
     want_main_area = False

orangecontrib/spark/widgets/spark_to_orange.py → orangecontrib/spark/widgets/data/spark_to_orange.py

@@ -12,7 +12,7 @@
 class OWSparkToOrange(widget.OWWidget):
     name = "to Orange"
     description = "Convert Spark dataframe to Orange Table"
-    icon = "icons/spark.ico"
+    icon = "../icons/spark.ico"
 
     inputs = [("Sparkdf", pyspark.sql.DataFrame, "get_input", widget.Default)]
     outputs = [("Table", Table, widget.Dynamic)]

orangecontrib/spark/widgets/spark_to_pandas.py → orangecontrib/spark/widgets/data/spark_to_pandas.py

@@ -1,33 +1,25 @@
 __author__ = 'jamh'
 
-from Orange.data import Table
-from PyQt4.QtGui import QSizePolicy
-from Orange.widgets import widget, gui
-from orangecontrib.spark.utils.bdutils import pandas_to_orange
 import pandas
-from Orange.widgets import widget, gui, settings
-from pyspark import SparkConf, SparkContext
 import pyspark
+from Orange.widgets import widget, gui, settings
+from PyQt4.QtGui import QSizePolicy
 
 
 class OWSparkToPandas(widget.OWWidget):
-    name = "to pandas"
+    name = "to Pandas"
     description = "Convert Spark dataframe to Pandas"
-    icon = "icons/spark.ico"
+    icon = "../icons/spark.ico"
 
-    inputs = [("Sparkdf", pyspark.sql.DataFrame, "get_input", widget.Default)]
+    inputs = [("DataFrame", pyspark.sql.DataFrame, "get_input", widget.Default)]
     outputs = [("Dataframe", pandas.DataFrame, widget.Dynamic)]
     settingsHandler = settings.DomainContextHandler()
 
-    NOTHING = "Nothing on input"
-
     def __init__(self):
         super().__init__()
         self.obj_type = self.NOTHING
         gui.label(self.controlArea, self, "Spark->Pandas:")
         self.setSizePolicy(QSizePolicy.Maximum, QSizePolicy.Maximum)
 
     def get_input(self, obj):
-        self.obj_type = self.NOTHING if obj is None else type(obj).__name__
-
-        self.send("Table", obj.toPandas())
+        self.send("Dataframe", obj.toPandas())