Finished problem 4.19

lasso_ivanov_Eout_C

@@ -0,0 +1,49 @@
+lambda,avg_Eout
+0.05,1.5317447791513013
+0.1,1.486408724276711
+0.15000000000000002,1.446374016007614
+0.2,1.4115095219528349
+0.25,1.3821323367931286
+0.30000000000000004,1.3585651232091227
+0.35000000000000003,1.3404351980006861
+0.4,1.3271059562582548
+0.45,1.3179146179715362
+0.5,1.3119976027393687
+0.55,1.3083836886052724
+0.6000000000000001,1.3059630877070922
+0.65,1.304177736211483
+0.7000000000000001,1.3026141807221838
+0.75,1.3008054063502135
+0.8,1.2987178133042698
+0.8500000000000001,1.2964309796844096
+0.9,1.294025006161774
+0.9500000000000001,1.2914677523961886
+1.0,1.288919568723812
+1.05,1.2863939233577153
+1.1,1.2838502176307174
+1.1500000000000001,1.2812941241917122
+1.2000000000000002,1.2787730249757496
+1.25,1.2763994450529057
+1.3,1.2741843443284109
+1.35,1.27186560029138
+1.4000000000000001,1.2695627379752767
+1.4500000000000002,1.2671351953414647
+1.5,1.264848152696345
+1.55,1.2623863771343193
+1.6,1.2598923192067706
+1.6500000000000001,1.2574752880237927
+1.7000000000000002,1.25509254674278
+1.75,1.2528959362757186
+1.8,1.2508394179921443
+1.85,1.248496660137221
+1.9000000000000001,1.246453247519282
+1.9500000000000002,1.244442083210317
+2.0,1.2424770668207166
+3.0,1.208241890741746
+4.0,1.1874877016930259
+5.0,1.1723934413667902
+6.0,1.1656608547398541
+7.0,1.1665532057491588
+8.0,1.1728224695893217
+9.0,1.1837406890939943
+10.0,1.2013004446398399

libs/linear_models.py

@@ -14,7 +14,7 @@
 import matplotlib.pyplot as plt
 from scipy.optimize import minimize
 from sklearn.preprocessing import normalize
-from sklearn.linear_model import Ridge
+from sklearn.linear_model import Ridge, Lasso
 
 import data_util as du
 
@@ -149,25 +149,29 @@ def calc_error(self, X, y):
 
 class LinearRegression(LinearRegressionBase):
     def __init__(self, algo_name, reg_type, reg_param, 
-                poly_degree = None):
+                poly_degree = None, solver=None):
         super().__init__(algo_name)
         self.reg_type = reg_type
         self.reg_param = reg_param
         self.poly_degree = poly_degree #If apply polynomial transformation first
+        self.solver = solver
 
 
     def fit(self, X, y):
         Z = X
         if self.poly_degree:
-            Z = du.polynomial_transform(self.poly_degree, X)        
+            Z = du.polynomial_transform(self.poly_degree, X)       
+
         if self.algo_name == 'lasso':
-            self.w = lasso_fit(Z, y, self.reg_param, self.reg_type)
+            self.w = lasso_fit(Z, y, self.reg_param, self.reg_type, self.solver)
         elif self.algo_name == 'ridge':
             self.w = ridge_fit(Z, y, self.reg_param, self.reg_type)
         elif self.algo_name == 'sklearn_lasso':
-            pass
+            self.w = sklearn_lasso(Z, y, self.reg_param)
         elif self.algo_name == 'sklearn_ridge':
             self.w = sklearn_ridge(Z, y, self.reg_param)
+        elif self.algo_name == 'sklearn_lr':
+            self.w = sklearn_linear_regression(Z, y)
         else:
             raise ValueError("Not implemented")
 
@@ -193,11 +197,25 @@ def sklearn_ridge(X, y, lambda_t, fit_intercept = False, solver='auto'):
     clf.fit(X, y)
     return clf.coef_.reshape(-1, 1)
 
+def sklearn_lasso(X, y, lambda_t, fit_intercept = False):
+    #If added 1 into features, set fit_intercept = False
+    #else, if you want to fit the intercept, set it to True
+    # This minimizes: \frac{1}{2N}|Xw-y|^2_2 + \lambda|w|_1
+    clf = Lasso(alpha=lambda_t, fit_intercept = fit_intercept)
+    clf.fit(X, y)
+    return clf.coef_.reshape(-1, 1)    
+
+def sklearn_linear_regression(X, y, fit_intercept = False):
+    clf = sklearn.linear_model.LinearRegression(fit_intercept)
+    clf.fit(X, y)
+    return clf.coef_.reshape(-1, 1)
+
 def lasso_fit_tikhonov(X, y, reg_param):
     raise ValueError("Not implemented")
 
-def lasso_fit_ivanov(X, y, reg_param):
+def lasso_fit_ivanov(X, y, reg_param, solver=None):
     # Apply quadratic programming to solve this
+    # solver is None or 'mosek'
     N = len(y) #number of samples
     d = X.shape[1]
     num_vars = 2*d
@@ -237,9 +255,9 @@ def lasso_fit_ivanov(X, y, reg_param):
     q = cvxopt.matrix(q)
     G = cvxopt.matrix(G)
     h = cvxopt.matrix(h)
-    res = cvxopt.solvers.qp(P, q, G, h, options={'show_progress':False})
+    res = cvxopt.solvers.qp(P, q, G, h, solver = solver, options={'show_progress':False})
     if res['status'] != 'optimal':
-        print("Couldn't find optimal solution")
+        print("Couldn't find optimal solution with reg_param: ", reg_param)
         print('Final status: ', res['status'])
     w = res['x']
     #print('w: ', type(w), w)
@@ -278,9 +296,9 @@ def calc_sum_of_squares(X, y, w):
 
 def calc_ss_with_constraints(X, y, w, C):
     # Compute the sum of squares and quadratic constraint
-    ss = calc_sum_of_squares(X, y, w)
+    ss = calc_sum_of_squares(X, y, w).flatten()
     constraint = np.matmul(w.transpose(), w) - C
-    res = np.array([ss, constraint]).reshape(2, 1) #2x1
+    res = np.array([ss, constraint.flatten()]).reshape(2, 1) #2x1
     return res
 
 def calc_derivative_ss(X, y, w):
@@ -294,18 +312,22 @@ def calc_derivative_ss(X, y, w):
 def calc_derivative_ss_with_constraints(X, y, w):
     dss = calc_derivative_ss(X, y, w) #(num_features +1)x1
     d_constraint = 2*w # (num_features + 1)x1
-    res = np.array([dss, d_constraint]).reshape(2, -1) # 2x(num_features +1)
+    dss = dss.transpose()
+    d_constraint = d_constraint.transpose()
+    res = np.vstack([dss, d_constraint]) # 2x(num_features +1)
     return res
 
 def calc_2nd_deriv_ss(X):
     N = X.shape[0]
-    XTX = np.matmul(X.transpose(), X)
+    # row#1: df1/dx1, df1/dx2, df1/dx3, ...
+    # row#2: df2/dx1, df2/dx2, df2/dx3, ...
+    XTX = np.matmul(X.transpose(), X).transpose()
     res = 2*XTX/N #(num_features+1) x (num_features+1)
     return res
 
 def calc_2nd_deriv_ss_with_constraints(X, z):
     deriv2_ss = z[0]*calc_2nd_deriv_ss(X)
-    deriv2_constraint = z[1] * 2
+    deriv2_constraint = z[1] * 2 * np.identity(X.shape[1])
     res = deriv2_ss + deriv2_constraint
     return res
 
@@ -318,6 +340,7 @@ def F(w = None, z = None):
         if w is None:
             w0 = np.zeros((num_features, 1)) #any feasible point
             return 1, cvxopt.matrix(w0)
+        w = np.array(w).reshape(num_features, 1)
         f = calc_ss_with_constraints(X, y, w, reg_param)        
         Df = calc_derivative_ss_with_constraints(X, y, w)
         f, Df = cvxopt.matrix(f), cvxopt.matrix(Df)
@@ -328,18 +351,18 @@ def F(w = None, z = None):
         return f, Df, H
     res = cvxopt.solvers.cp(F, options={'show_progress':False})
     if res['status'] != 'optimal':
-        print("Couldn't find optimal solution")
+        print("Couldn't find optimal solution with reg_param: ", reg_param)
         print('Final status: ', res['status'])
     w = res['x']
     w = np.array(w)
     return w
 
-def lasso_fit(X, y, reg_param, reg_type):
+def lasso_fit(X, y, reg_param, reg_type, solver):
     w = None
     if reg_type == 'Tikhonov':
         w = lasso_fit_tikhonov(X, y, reg_param)
     elif reg_type == 'Ivanov':
-        w = lasso_fit_ivanov(X, y, reg_param)
+        w = lasso_fit_ivanov(X, y, reg_param, solver)
     else:
         raise ValueError("Not implemented")
     return w

ridge_ivanov_Eout_C

@@ -0,0 +1,49 @@
+lambda,avg_Eout
+0.05,1.3399893220247276
+0.1,1.2759927226510948
+0.15000000000000002,1.23950929949807
+0.2,1.2160508872282343
+0.25,1.2000510576914247
+0.30000000000000004,1.188705694033259
+0.35000000000000003,1.180403510423792
+0.4,1.1741479643486348
+0.45,1.1692887080045546
+0.5,1.1653868273677364
+0.55,1.1621457250369058
+0.6000000000000001,1.1593684103170598
+0.65,1.1569202724769247
+0.7000000000000001,1.1547052440822432
+0.75,1.152655099419062
+0.8,1.1507229738227633
+0.8500000000000001,1.148877542884888
+0.9,1.1470981536132059
+0.9500000000000001,1.145371599733018
+1.0,1.1436901357162617
+1.05,1.1420497513691694
+1.1,1.140448489758452
+1.1500000000000001,1.1388854484515212
+1.2000000000000002,1.1373601796109092
+1.25,1.1358722002240773
+1.3,1.1344208295858584
+1.35,1.133005300222508
+1.4000000000000001,1.1316249373498715
+1.4500000000000002,1.1302792949564073
+1.5,1.1289681618275687
+1.55,1.1276914522446106
+1.6,1.1264490611224844
+1.6500000000000001,1.1252407597200202
+1.7000000000000002,1.124066160905204
+1.75,1.122924745482796
+1.8,1.121815928707776
+1.85,1.1207391376959985
+1.9000000000000001,1.1196938774200897
+1.9500000000000002,1.118679756373862
+2.0,1.1176964757957029
+3.0,1.103935506073491
+4.0,1.098660488110628
+5.0,1.0982779692983635
+6.0,1.100840549837464
+7.0,1.1054018410894817
+8.0,1.1113699321385717
+9.0,1.1183459014763026
+10.0,1.1260424025467393