Jure Žbontar avatar Jure Žbontar committed 399c47d

Softmax fix.

Comments (0)

Files changed (2)

mlclass/logistic_regression.py

     def predict(self, X):
         return sigmoid(X.dot(self.theta))
 
+
 class SoftmaxRegressionGD:
     def __init__(self, lambda_=1, **fmin_args):
         self.lambda_ = lambda_
 
     def cost_grad(self, Theta_flat):
         m, n = self.X.shape
-        k = self.Y.shape[1]
 
-        Theta = Theta_flat.reshape((k, n))
+        Theta = Theta_flat.reshape((self.num_classes, n))
 
         P = np.exp(self.X.dot(Theta.T))
         P /= np.sum(P, axis=1)[:,None]
 
-        j = -np.sum(np.log(P) * self.Y)
+        j = -np.sum(np.log(P[xrange(self.y.size),self.y]))
         j += self.lambda_ * Theta_flat.dot(Theta_flat) / 2.0
         j /= m
 
         grad = self.X.T.dot(P - self.Y).T
-
         grad += self.lambda_ * Theta
         grad /= m
 
         return j, grad.ravel()
 
-    def fit(self, X, Y):
-        self.X, self.Y = X, Y
-        theta = np.zeros(Y.shape[1] * X.shape[1])
+    def fit(self, X, y):
+        m, n = X.shape
+        self.num_classes = y.max() + 1
+
+        self.X, self.y = X, y
+        self.Y = np.eye(self.num_classes)[self.y]
+
+        theta = np.zeros(self.num_classes * n)
         theta, j, ret = fmin_l_bfgs_b(self.cost_grad, theta, **self.fmin_args)
         if ret['warnflag'] != 0:
             warnings.warn('L-BFGS failed to converge')
-        self.Theta = theta.reshape((Y.shape[1], X.shape[1]))
+        self.Theta = theta.reshape((self.num_classes, X.shape[1]))
 
     def predict(self, X):
         P = np.exp(X.dot(self.Theta.T))
         P /= np.sum(P, axis=1)[:,None]
-
-        return np.argmax(P, axis=1)
+        return P
 
 
 if __name__ == '__main__':
         from data import load_svm
 
         X, y = load_svm('../data/iris.scale')
-        Y = np.eye(3)[(y - 1 + 0.5).astype(np.int)]
-        m = SoftmaxRegressionGD(lambda_=0)
-        m.fit(X, Y)
-        print np.mean(m.predict(X) == (y - 1 + 0.5).astype(np.int))
+        y = (y - 1).astype(np.int)
+
+        m = SoftmaxRegressionGD(lambda_=1)
+        m.fit(X, y)
+        print np.mean(np.argmax(m.predict(X), axis=1) == y)
 
     softmax()
 

mlclass/neural_networks.py

 from scipy.optimize import fmin_l_bfgs_b
 
 class NeuralNetwork:
-    def __init__(self, layers,lambda_=1, callback=None, **fmin_args):
+    def __init__(self, layers, lambda_=1, callback=None, **fmin_args):
         self.layers = layers
         self.lambda_ = lambda_
         self.callback = callback
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.