Fixed critical bug in logistic/linear regression with validation trac…

…king (#532)

* Fixed critical bug in logistic/linear regression when:
- Validation set is turned on
- Large number of features are present

This is escpecially a problem with image classification with large #
classes.

The second order information is being computed but never used. This is
an O(N * F * F * C * C) operation where F is the number of features
(which is around 1K) and N is the size of the validation set. This is
really, really expensive. This quadratic complexity is no longer needed.

* Fix to make the unit test pass.

src/unity/python/turicreate/test/test_image_classifier.py

@@ -20,7 +20,7 @@
 import numpy as np
 import platform
 
-def _get_data():
+def _get_data(num_examples = 100):
     from PIL import Image as _PIL_Image
     rs = np.random.RandomState(1234)
     _format = {'JPG': 0, 'PNG': 1, 'RAW': 2, 'UNDEFINED': 3}
@@ -46,7 +46,6 @@ def from_pil_image(pil_img):
                 _image_data_size=image_data_size)
         return img
 
-    num_examples = 100
     images = []
     random_labels = [rs.randint(0,5) for i in range(num_examples)]
     for i in range(num_examples):
@@ -64,14 +63,14 @@ def from_pil_image(pil_img):
 
 class ImageClassifierTest(unittest.TestCase):
     @classmethod
-    def setUpClass(self, model='resnet-50', input_image_shape=(3, 224, 224), tol=0.02):
+    def setUpClass(self, model='resnet-50', input_image_shape=(3, 224, 224), tol=0.02, num_examples = 100):
         self.feature = 'awesome_image'
         self.target = 'awesome_label'
         self.input_image_shape = input_image_shape
         self.pre_trained_model = model
         self.tolerance = tol
 
-        self.sf = _get_data()
+        self.sf = _get_data(num_examples)
         self.model = tc.image_classifier.create(self.sf, target=self.target,
                                                 model=self.pre_trained_model,
                                                 seed=42)
@@ -246,6 +245,13 @@ def setUpClass(self):
                                                               input_image_shape=(3, 227, 227),
                                                               tol=0.005)
 
+class ImageClassifierLargerSqueezeNetTest(ImageClassifierTest):
+    @classmethod
+    def setUpClass(self):
+        super(ImageClassifierLargerSqueezeNetTest, self).setUpClass(model='squeezenet_v1.1',
+                                                              input_image_shape=(3, 227, 227),
+                                                              tol=0.005, num_examples = 200)
+
 
 @unittest.skipIf(tc.util._num_available_gpus() == 0, 'Requires GPU')
 @pytest.mark.gpu

src/unity/toolkits/supervised_learning/linear_regression.cpp

@@ -307,8 +307,8 @@ void linear_regression::train(){
   if (lr_interface->num_validation_examples() > 0) {
     // Recycle lvalues from stats to use as out parameters here, now that we're
     // otherwise done reading from stats.
-    lr_interface->compute_validation_second_order_statistics(
-        stats.solution, stats.hessian, stats.gradient, stats.func_value);
+    lr_interface->compute_validation_first_order_statistics(
+        stats.solution, stats.gradient, stats.func_value);
     state["validation_loss"] =  stats.func_value;
     state["validation_rmse"] =  sqrt((stats.func_value)/examples);
   }

src/unity/toolkits/supervised_learning/linear_regression_opt_interface.cpp

@@ -141,8 +141,8 @@ std::vector<std::string> linear_regression_opt_interface::get_status(
  * Compute the first order statistics
 */
 void linear_regression_opt_interface::compute_first_order_statistics(const
-    DenseVector& point, DenseVector& gradient, double& function_value, const
-    size_t mbStart, const size_t mbSize) {
+    ml_data& data, const DenseVector& point, DenseVector& gradient, double&
+    function_value, const size_t mbStart, const size_t mbSize) {
   DASSERT_TRUE(mbStart == 0);
   DASSERT_TRUE(mbSize == (size_t)(-1));
 
@@ -218,9 +218,9 @@ void linear_regression_opt_interface::compute_first_order_statistics(const
 /**
  * Compute the second order statistics
 */
-void linear_regression_opt_interface::compute_second_order_statistics(
-    const ml_data& data, const DenseVector& point, DenseMatrix& hessian,
-    DenseVector& gradient, double& function_value) {
+void linear_regression_opt_interface::compute_second_order_statistics( const
+    DenseVector& point, DenseMatrix& hessian, DenseVector& gradient, double&
+    function_value) {
 
   std::vector<DenseMatrix> H(n_threads, 
                         arma::zeros(variables,variables));
@@ -303,19 +303,18 @@ void linear_regression_opt_interface::compute_second_order_statistics(
   }
 }
 
-void linear_regression_opt_interface::compute_second_order_statistics(
-    const DenseVector& point, DenseMatrix& hessian, DenseVector& gradient,
-    double& function_value) {
-  compute_second_order_statistics(
-      data, point, hessian, gradient, function_value);
+void linear_regression_opt_interface::compute_first_order_statistics(const
+    DenseVector& point, DenseVector& gradient, double& function_value, const
+    size_t mbStart, const size_t mbSize) {
+  compute_first_order_statistics(
+      data, point, gradient, function_value, mbStart, mbSize);
 }
 
 void
-linear_regression_opt_interface::compute_validation_second_order_statistics(
-    const DenseVector& point, DenseMatrix& hessian, DenseVector& gradient,
-    double& function_value) {
-  compute_second_order_statistics(
-      valid_data, point, hessian, gradient, function_value);
+linear_regression_opt_interface::compute_validation_first_order_statistics(
+    const DenseVector& point, DenseVector& gradient, double& function_value) {
+  compute_first_order_statistics(
+      valid_data, point, gradient, function_value);
 }
 
 } // supervised

src/unity/toolkits/supervised_learning/linear_regression_opt_interface.hpp

@@ -158,24 +158,22 @@ class linear_regression_opt_interface: public
       hessian, DenseVector& gradient, double & function_value);
 
   /**
-   * Compute second order statistics at the given point with respect to the
+   * Compute first order statistics at the given point with respect to the
    * validation data. (Gradient & Function value)
    *
    * \param[in]  point           Point at which we are computing the stats.
-   * \param[out] hessian         Hessian (Dense)
    * \param[out] gradient        Dense gradient
    * \param[out] function_value  Function value
    *
    */
-  void compute_validation_second_order_statistics(
-      const DenseVector& point, DenseMatrix& hessian, DenseVector& gradient,
-      double &function_value);
+  void compute_validation_first_order_statistics(
+      const DenseVector& point, DenseVector& gradient, double &function_value);
 
   private:
 
-  void compute_second_order_statistics(
-      const ml_data& data, const DenseVector &point, DenseMatrix& hessian,
-      DenseVector& gradient, double & function_value);
+  void compute_first_order_statistics(const ml_data& data, const DenseVector
+      &point, DenseVector& gradient, double & function_value, const size_t
+      mbStart = 0, const size_t mbSize = -1);
 };
 
 

src/unity/toolkits/supervised_learning/logistic_regression.cpp

@@ -340,8 +340,8 @@ void logistic_regression::train() {
   if (lr_interface->num_validation_examples() > 0) {
     // Recycle lvalues from stats to use as out parameters here, now that we're
     // otherwise done reading from stats.
-    lr_interface->compute_validation_second_order_statistics(
-        stats.solution, stats.hessian, stats.gradient, stats.func_value);
+    lr_interface->compute_validation_first_order_statistics(
+        stats.solution, stats.gradient, stats.func_value);
     state["validation_loss"] =  stats.func_value;
   }
 

src/unity/toolkits/supervised_learning/logistic_regression_opt_interface.cpp

@@ -201,9 +201,9 @@ std::vector<std::string> logistic_regression_opt_interface::get_status(
 /**
  * Compute the first order statistics
 */
-void logistic_regression_opt_interface::compute_first_order_statistics(const
-    DenseVector& point, DenseVector& gradient, double& function_value, const
-    size_t mbStart, const size_t mbSize) {
+void logistic_regression_opt_interface::compute_first_order_statistics(
+    const ml_data& data, const DenseVector& point, DenseVector& gradient,
+    double& function_value, const size_t mbStart, const size_t mbSize) {
   DASSERT_TRUE(mbStart == 0);
   DASSERT_TRUE(mbSize == (size_t)(-1));
 
@@ -304,8 +304,8 @@ void logistic_regression_opt_interface::compute_first_order_statistics(const
  * Compute the second order statistics
 */
 void logistic_regression_opt_interface::compute_second_order_statistics(
-    const ml_data& data, const DenseVector& point, DenseMatrix& hessian,
-    DenseVector& gradient, double& function_value) {
+    const DenseVector& point, DenseMatrix& hessian, DenseVector& gradient,
+    double& function_value) {
 
   timer t;
   double start_time = t.current_time();
@@ -438,19 +438,18 @@ void logistic_regression_opt_interface::compute_second_order_statistics(
                       << (t.current_time() - start_time) << "s" << std::endl; 
 }
 
-void logistic_regression_opt_interface::compute_second_order_statistics(
-    const DenseVector& point, DenseMatrix& hessian, DenseVector& gradient,
-    double& function_value) {
-  compute_second_order_statistics(
-      data, point, hessian, gradient, function_value);
+void logistic_regression_opt_interface::compute_first_order_statistics(const
+    DenseVector& point, DenseVector& gradient, double& function_value, const
+    size_t mbStart, const size_t mbSize) {
+  compute_first_order_statistics(
+      data, point, gradient, function_value, mbStart, mbSize);
 }
 
 void
-logistic_regression_opt_interface::compute_validation_second_order_statistics(
-    const DenseVector& point, DenseMatrix& hessian, DenseVector& gradient,
-    double& function_value) {
-  compute_second_order_statistics(
-      valid_data, point, hessian, gradient, function_value);
+logistic_regression_opt_interface::compute_validation_first_order_statistics(
+    const DenseVector& point, DenseVector& gradient, double& function_value) {
+  compute_first_order_statistics(
+      valid_data, point, gradient, function_value);
 }
 
 

src/unity/toolkits/supervised_learning/logistic_regression_opt_interface.hpp

@@ -183,24 +183,22 @@ class logistic_regression_opt_interface: public
       hessian, DenseVector& gradient, double & function_value);
 
   /**
-   * Compute second order statistics at the given point with respect to the
+   * Compute first order statistics at the given point with respect to the
    * validation data. (Gradient & Function value)
    *
    * \param[in]  point           Point at which we are computing the stats.
-   * \param[out] hessian         Hessian (Dense)
    * \param[out] gradient        Dense gradient
    * \param[out] function_value  Function value
    *
    */
-  void compute_validation_second_order_statistics(
-      const DenseVector& point, DenseMatrix& hessian, DenseVector& gradient,
-      double &function_value);
+  void compute_validation_first_order_statistics(
+      const DenseVector& point, DenseVector& gradient, double &function_value);
 
   private:
 
-  void compute_second_order_statistics(
-      const ml_data& data,const DenseVector& point, DenseMatrix& hessian,
-      DenseVector& gradient, double &function_value);
+  void compute_first_order_statistics(const ml_data& data, const DenseVector
+      &point, DenseVector& gradient, double & function_value, const size_t
+      mbStart = 0, const size_t mbSize = -1);
 };