forked from dclambert/Python-ELM
-
Notifications
You must be signed in to change notification settings - Fork 0
/
plot_elm_comparison.py
213 lines (155 loc) · 6.89 KB
/
plot_elm_comparison.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
======================
ELM Classifiers Comparison
======================
A comparison of a several ELMClassifiers with different types of hidden
layer activations.
ELMClassifier is a classifier based on the Extreme Learning Machine,
a single layer feedforward network with random hidden layer components
and least squares fitting of the hidden->output weights by default [1][2]
The point of this example is to illustrate the nature of decision boundaries
with different hidden layer activation types and regressors.
This should be taken with a grain of salt, as the intuition conveyed by
these examples does not necessarily carry over to real datasets.
In particular in high dimensional spaces data can more easily be separated
linearly and the simplicity of classifiers such as naive Bayes and linear SVMs
might lead to better generalization.
The plots show training points in solid colors and testing points
semi-transparent. The lower right shows the classification accuracy on the test
set.
References
__________
.. [1] http://www.extreme-learning-machines.org
.. [2] G.-B. Huang, Q.-Y. Zhu and C.-K. Siew, "Extreme Learning Machine:
Theory and Applications", Neurocomputing, vol. 70, pp. 489-501,
2006.
===============================================================================
Basis Functions:
gaussian rbf : exp(-gamma * (||x-c||/r)^2)
tanh : np.tanh(a)
sinsq : np.power(np.sin(a), 2.0)
tribas : np.clip(1.0 - np.fabs(a), 0.0, 1.0)
hardlim : np.array(a > 0.0, dtype=float)
where x : input pattern
a : dot_product(x, c) + b
c,r : randomly generated components
Label Legend:
ELM(10,tanh) :10 tanh units
ELM(10,tanh,LR) :10 tanh units, LogisticRegression
ELM(10,sinsq) :10 sin*sin units
ELM(10,tribas) :10 tribas units
ELM(10,hardlim) :10 hardlim units
ELM(20,rbf(0.1)) :20 rbf units gamma=0.1
"""
print __doc__
# Code source: Gael Varoqueux
# Andreas Mueller
# Modified for Documentation merge by Jaques Grobler
# Modified for Extreme Learning Machine Classifiers by David Lambert
# License: BSD
import numpy as np
import pylab as pl
from matplotlib.colors import ListedColormap
from sklearn.datasets import make_moons, make_circles, make_classification
from sklearn.preprocessing import StandardScaler
from sklearn.cross_validation import train_test_split
from sklearn.linear_model import LogisticRegression
from elm import GenELMClassifier
from random_layer import RBFRandomLayer, MLPRandomLayer
def get_data_bounds(X):
h = .02 # step size in the mesh
x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5
y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5
xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
np.arange(y_min, y_max, h))
return (x_min, x_max, y_min, y_max, xx, yy)
def plot_data(ax, X_train, y_train, X_test, y_test, xx, yy):
cm = ListedColormap(['#FF0000', '#0000FF'])
# Plot the training points
ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm)
# and testing points
ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm, alpha=0.6)
ax.set_xlim(xx.min(), xx.max())
ax.set_ylim(yy.min(), yy.max())
ax.set_xticks(())
ax.set_yticks(())
def plot_contour(ax, X_train, y_train, X_test, y_test, xx, yy, Z):
cm = pl.cm.RdBu
cm_bright = ListedColormap(['#FF0000', '#0000FF'])
ax.contourf(xx, yy, Z, cmap=cm, alpha=.8)
# Plot also the training points
ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright)
# and testing points
ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm_bright, alpha=0.6)
ax.set_xlim(xx.min(), xx.max())
ax.set_ylim(yy.min(), yy.max())
ax.set_xticks(())
ax.set_yticks(())
ax.set_title(name)
ax.text(xx.max() - 0.3, yy.min() + 0.3, ('%.2f' % score).lstrip('0'),
size=13, horizontalalignment='right')
def make_datasets():
return [make_moons(n_samples=200, noise=0.3, random_state=0),
make_circles(n_samples=200, noise=0.2, factor=0.5, random_state=1),
make_linearly_separable()]
def make_classifiers():
names = ["ELM(10,tanh)", "ELM(10,tanh,LR)", "ELM(10,sinsq)",
"ELM(10,tribas)", "ELM(hardlim)", "ELM(20,rbf(0.1))"]
nh = 10
# pass user defined transfer func
sinsq = (lambda x: np.power(np.sin(x), 2.0))
srhl_sinsq = MLPRandomLayer(n_hidden=nh, activation_func=sinsq)
# use internal transfer funcs
srhl_tanh = MLPRandomLayer(n_hidden=nh, activation_func='tanh')
srhl_tribas = MLPRandomLayer(n_hidden=nh, activation_func='tribas')
srhl_hardlim = MLPRandomLayer(n_hidden=nh, activation_func='hardlim')
# use gaussian RBF
srhl_rbf = RBFRandomLayer(n_hidden=nh*2, rbf_width=0.1, random_state=0)
log_reg = LogisticRegression()
classifiers = [GenELMClassifier(hidden_layer=srhl_tanh),
GenELMClassifier(hidden_layer=srhl_tanh, regressor=log_reg),
GenELMClassifier(hidden_layer=srhl_sinsq),
GenELMClassifier(hidden_layer=srhl_tribas),
GenELMClassifier(hidden_layer=srhl_hardlim),
GenELMClassifier(hidden_layer=srhl_rbf)]
return names, classifiers
def make_linearly_separable():
X, y = make_classification(n_samples=200, n_features=2, n_redundant=0,
n_informative=2, random_state=1,
n_clusters_per_class=1)
rng = np.random.RandomState(2)
X += 2 * rng.uniform(size=X.shape)
return (X, y)
###############################################################################
datasets = make_datasets()
names, classifiers = make_classifiers()
i = 1
figure = pl.figure(figsize=(18, 9))
# iterate over datasets
for ds in datasets:
# preprocess dataset, split into training and test part
X, y = ds
X = StandardScaler().fit_transform(X)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.4,
random_state=0)
x_min, x_max, y_min, y_max, xx, yy = get_data_bounds(X)
# plot dataset first
ax = pl.subplot(len(datasets), len(classifiers) + 1, i)
plot_data(ax, X_train, y_train, X_test, y_test, xx, yy)
i += 1
# iterate over classifiers
for name, clf in zip(names, classifiers):
ax = pl.subplot(len(datasets), len(classifiers) + 1, i)
clf.fit(X_train, y_train)
score = clf.score(X_test, y_test)
# Plot the decision boundary. For that, we will asign a color to each
# point in the mesh [x_min, m_max]x[y_min, y_max].
Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])
# Put the result into a color plot
Z = Z.reshape(xx.shape)
plot_contour(ax, X_train, y_train, X_test, y_test, xx, yy, Z)
i += 1
figure.subplots_adjust(left=.02, right=.98)
pl.show()