Jure Žbontar avatar Jure Žbontar committed fb1abba

Initial commit.

Comments (0)

Files changed (3)

+syntax: glob
+
+*.pyc
+*.swp
+__pycache__
+import logging
+
+import numpy as np
+import scipy.sparse as sp
+from scipy.optimize import fmin_l_bfgs_b
+
+
+### utils ###
+def append_ones(X):
+    if sp.issparse(X):
+        return sp.hstack((np.ones((X.shape[0], 1)), X)).tocsr()
+    else:
+        return np.hstack((np.ones((X.shape[0], 1)), X))
+
+
+def sigmoid(x):
+    return 1.0 / (1.0 + np.exp(-x))
+
+
+def sigmoid_gradient(x):
+    sx = sigmoid(x)
+    return sx * (1 - sx)
+
+
+### Linear Regression ###
+class LinearRegression:
+    def __init__(self, lambda_=1, **fmin_args):
+        self.lambda_ = lambda_
+        self.fmin_args = fmin_args
+
+    def cost_grad(self, theta, X, y):
+        t = X.dot(theta) - y
+
+        cost = t.dot(t)
+        cost += self.lambda_ * theta.dot(theta) / 2
+        cost /= X.shape[0]
+
+        grad = X.T.dot(t)
+        grad += self.lambda_ * theta
+        grad /= X.shape[0]
+
+        return cost, grad
+
+    def fit(self, X, y):
+        assert np.issubdtype(y.dtype, float)
+        assert y.ndim == 1
+
+        theta = np.zeros(X.shape[1])
+        self.theta, cost, ret = fmin_l_bfgs_b(
+                self.cost_grad, theta, args=(X, y), **self.fmin_args)
+        if ret['warnflag'] != 0:
+            logging.warning('L-BFGS failed to converge')
+
+    def predict(self, X):
+        return X.dot(self.theta)
+
+    def fit_predict(self, X, y, X_test):
+        self.fit(X, y)
+        return self.predict(X_test)
+
+
+### Logistic Regression ###
+class LogisticRegression:
+    def __init__(self, lambda_=1, **fmin_args):
+        self.lambda_ = lambda_
+        self.fmin_args = fmin_args
+
+    def cost_grad(self, theta, X, y):
+        sx = sigmoid(X.dot(theta))
+
+        cost = -np.log(np.where(y, sx, 1 - sx)).sum()
+        cost += self.lambda_ * theta.dot(theta)
+        cost /= X.shape[0]
+
+        grad = X.T.dot(sx - y)
+        grad += self.lambda_ * theta
+        grad /= X.shape[0]
+
+        return cost, grad
+
+    def fit(self, X, y):
+        assert list(np.unique(y).astype(int)) == [0, 1]
+        assert y.ndim == 1
+
+        theta = np.zeros(X.shape[1])
+        self.theta, cost, ret = fmin_l_bfgs_b(
+                self.cost_grad, theta, args=(X, y), **self.fmin_args)
+        if ret['warnflag'] != 0:
+            logging.warning('L-BFGS failed to converge')
+
+    def predict(self, X):
+        return sigmoid(X.dot(self.theta))
+
+    def fit_predict(self, X, y, X_test):
+        self.fit(X, y)
+        return self.predict(X_test)
+
+
+### Multilayer Perceptron Classifier ###
+class MLPClassifier:
+    def __init__(self, num_hidden, lambda_=1, callback=None, **fmin_args):
+        self.num_hidden = num_hidden
+        self.lambda_ = lambda_
+        self.callback = callback
+        self.fmin_args = fmin_args
+
+    def unfold_params(self, params):
+        i0, i1, i2 = self.layers
+        i = (i0 + 1) * i1
+        Theta1 = params[:i].reshape((i1, i0 + 1))
+        Theta2 = params[i:].reshape((i2, i1 + 1))
+        return Theta1, Theta2
+
+    def cost_grad(self, params, X, y):
+        Theta1, Theta2 = self.unfold_params(params)
+
+        if self.callback:
+            self.Theta1 = Theta1
+            self.Theta2 = Theta2
+            self.callback(self)
+
+        # feedforward propagation
+        a1 = X
+        z2 = a1.dot(Theta1.T)
+        a2 = append_ones(sigmoid(z2))
+        z3 = a2.dot(Theta2.T)
+        a3 = sigmoid(z3)
+
+        # cost
+        cost = -np.log(np.where(y, a3, 1 - a3)).sum()
+
+        t1 = Theta1.copy()
+        t1[:, 0] = 0
+        t2 = Theta2.copy()
+        t2[:, 0] = 0
+
+        # regularization
+        reg = np.dot(t1.flat, t1.flat)
+        reg += np.dot(t2.flat, t2.flat)
+        cost += float(self.lambda_) * reg / 2 
+        cost /= X.shape[0]
+
+        # gradient
+        d3 = a3 - y
+        d2 = d3.dot(Theta2)[:, 1:] * sigmoid_gradient(z2)
+
+        D2 = a2.T.dot(d3).T
+        D1 = a1.T.dot(d2).T
+
+        # regularization
+        D2 += t2 * self.lambda_
+        D1 += t1 * self.lambda_
+
+        grad = np.hstack((D1.flat, D2.flat)) / X.shape[0]
+
+        return cost, grad
+
+    def fit(self, X, y):
+        self.layers = [X.shape[1], self.num_hidden, y.shape[1]]
+        i0, i1, i2 = self.layers
+
+        n_params = i1 * (i0 + 1) + i2 * (i1 + 1)
+        eps = np.sqrt(6) / np.sqrt(i0 + i2)
+        params = np.random.randn(n_params) * 2 * eps - eps
+
+        X = append_ones(X)
+        self.thetas, _, _ = fmin_l_bfgs_b(
+            self.cost_grad, params, args=(X, y), **self.fmin_args)
+
+    def predict(self, X):
+        m, n = X.shape
+        Theta1, Theta2 = self.unfold_params(self.thetas)
+        a2 = sigmoid(append_ones(X).dot(self.Theta1.T))
+        a3 = sigmoid(np.column_stack((np.ones(m), a2)).dot(self.Theta2.T))
+
+        return a3
+
+    def fit_predict(self, X, y, X_test):
+        self.fit(X, y)
+        return self.predict(X_test)
+
+import unittest
+
+import numpy as np
+import scipy.sparse as sp
+from sklearn.datasets import make_regression, make_classification
+
+import ml
+
+class TestLinearRegression(unittest.TestCase):
+    def setUp(self):
+        self.X, self.y = make_regression(random_state=42)
+        self.model = ml.LinearRegression(1)
+        self.theta = np.array([
+            -4.77172827e-01,   2.90190165e+01,   3.18874226e+00,
+             6.70257002e+01,   1.28337188e+01,  -3.22351307e-01,
+            -1.13026678e+00,   3.65631080e-01,  -1.30581397e+00,
+             4.07956863e-01,  -4.76318730e-01,  -2.15700752e+00,
+             7.15628146e-01,  -1.57455382e+00,  -3.21417567e+00,
+            -2.55121806e+00,  -1.58359504e+00,   2.13822874e+00,
+             1.30537270e+00,   8.11870876e-01,  -2.07253541e+00,
+            -3.68228686e+00,   1.23929938e+00,  -9.89378327e-01,
+            -1.79314740e+00,   1.22307194e+00,   8.05742223e-02,
+             2.50289885e+00,  -9.67468265e-01,   7.78784191e-01,
+             4.98640316e+00,   1.07223202e+00,  -1.92524799e+00,
+            -1.68253736e+00,  -3.34671595e+00,  -4.59259248e-01,
+            -4.51391286e-01,   1.81771110e+00,  -4.38359687e+00,
+            -9.71120400e-01,   5.05796214e-01,  -1.02276449e-01,
+            -6.92621725e-01,  -4.62451125e-01,   7.52533982e+00,
+            -9.07811089e-02,   8.44350186e-02,  -2.25134222e+00,
+            -1.27012811e+00,  -1.61968431e+00,   8.12125816e-01,
+             1.12088013e+00,   1.88234299e+00,   5.88837468e+00,
+             1.82962280e-01,   1.36431663e+00,  -1.82681084e+00,
+            -2.81596215e+00,  -4.24640531e-02,  -2.79808976e+00,
+             1.60144097e+00,   1.53982899e+00,  -7.49346699e-01,
+             2.13821779e-01,   1.35181441e+00,  -1.80690384e+00,
+            -3.47583372e+00,  -1.24017317e+00,  -2.45470698e-02,
+             1.54606366e+00,   8.95729408e+00,  -1.61228698e+00,
+             1.82014001e+00,   5.47769871e+00,   1.65355877e-01,
+            -4.20784063e+00,   3.44778686e+01,   5.52353938e-01,
+             2.28040257e+00,  -1.21284831e+00,  -1.91238992e+00,
+             1.31319324e+00,   3.09844498e-01,  -2.30970531e+00,
+             1.04017432e+00,   9.04800893e-01,  -2.87455005e-01,
+             3.36829884e+01,   7.06362134e+01,   1.85591741e+00,
+             1.87920864e+00,   1.90346581e+00,  -2.00778104e-01,
+            -1.39961185e+00,   5.49357247e+01,   1.56533752e+00,
+             2.47082164e+00,  -8.54779577e-01,   1.72785735e+00,
+             6.57696489e-01])
+
+    def test_dense(self):
+        self.model.fit(self.X, self.y)
+        self.assertTrue(np.allclose(self.model.theta, self.theta))
+
+    def test_sparse(self):
+        X = sp.csr_matrix(self.X)
+        self.model.fit(X, self.y)
+        self.assertTrue(np.allclose(self.model.theta, self.theta))
+
+
+class TestLogisticRegression(unittest.TestCase):
+    def setUp(self):
+        self.X, self.y = make_classification(n_samples=1000, random_state=42)
+        self.model = ml.LogisticRegression(1)
+        self.theta = np.array([
+             0.09802625,  2.34418976,  0.24921228, -0.01221083,  0.19087675,
+            -0.04439369, -0.2981486 , -0.67829338, -0.13809016, -0.21804709,
+             0.1378354 ,  0.13347275,  2.84678878,  0.03970425,  0.14784195,
+             0.08251049, -0.1605976 ,  0.16705051,  0.0726403 ,  0.14510616])
+
+    def test_dense(self):
+        self.model.fit(self.X, self.y)
+        self.assertTrue(np.allclose(self.model.theta, self.theta))
+
+    def test_sparse(self):
+        X = sp.csr_matrix(self.X)
+        self.model.fit(X, self.y)
+        self.assertTrue(np.allclose(self.model.theta, self.theta))
+
+
+class TestMLPClassifier(unittest.TestCase):
+    def setUp(self):
+        self.X, self.y = make_classification(
+            n_samples=1000, n_classes=3, n_informative=3, random_state=42)
+        self.y = np.eye(3)[self.y.astype(int)]
+
+        self.model = ml.MLPClassifier(5, 1)
+        self.thetas = np.array([
+            -5.93815827e+00,  -1.97161444e+00,  -6.06801933e-02,
+             2.08215270e-01,   3.65040044e-01,  -1.46861900e+00,
+            -1.64292038e-01,  -4.76303294e-02,  -4.14165093e-01,
+             6.46592915e-02,  -5.56841157e-02,  -1.22269658e+00,
+             3.72485314e+00,  -2.38345730e-01,   3.42925652e-03,
+             2.31706314e-01,   2.04536207e-01,  -8.28730331e-02,
+             6.34311173e-01,  -2.60754383e+00,  -2.14819329e-02,
+            -3.02620667e+00,   4.70404919e-01,  -3.87846883e-01,
+            -1.46772891e-01,   5.87527956e-02,  -3.16688761e+00,
+            -4.08493942e-02,  -3.51043100e-02,  -3.18871989e-01,
+            -1.69292157e-01,   5.30932040e-02,   4.38617467e-01,
+            -4.70686849e+00,  -1.04133437e-01,  -1.75652183e-01,
+            -1.24870384e-01,  -1.94327887e-01,  -3.00977224e-01,
+             1.77798812e-01,   1.11611391e+00,  -1.66792009e-01,
+            -4.09303054e+00,  -9.61537394e-02,   3.04046794e-02,
+            -4.46869203e-03,  -2.49189370e-01,   3.39921934e+00,
+             2.45317113e-01,  -4.28793598e-01,   1.18697719e-01,
+             1.68259816e-01,  -3.63286700e-01,  -5.63580699e-01,
+             4.71783241e+00,  -3.68411077e-02,   1.49940080e-01,
+            -3.08358310e-01,   3.79292026e-02,   2.90174217e-01,
+            -2.13066428e-01,  -7.10440628e-01,  -1.33672059e-01,
+            -1.95411636e+00,   1.15106838e+00,   3.48334771e-01,
+            -3.07905587e-01,  -5.05776087e-01,  -1.14262023e+00,
+            -6.32408784e-01,  -6.55329377e-01,  -4.38418342e-01,
+             2.62516023e-01,  -6.67562239e-01,  -1.38693968e+00,
+             3.26847111e-01,  -1.06572369e+00,   1.31436889e-01,
+             5.57602539e-01,  -5.29805592e-01,  -1.68772530e-02,
+             2.01179717e-01,   1.22148574e+00,  -5.79943336e-01,
+             3.44090730e-01,  -2.11467533e+00,   1.23938978e-01,
+             2.44605091e-01,   6.09585841e-01,  -1.18612626e+00,
+            -1.32646880e-01,   5.64245570e-01,  -9.36997013e-02,
+             3.85525520e-02,  -7.33324947e-02,  -1.31468230e-01,
+             1.86029898e+00,  -2.88373647e-01,  -2.58567421e-04,
+            -8.37320651e-02,   3.13679550e-01,   7.12594893e-02,
+            -3.27953216e-01,  -2.53328494e+00,  -3.15342578e-01,
+            -1.59178250e+00,  -2.79131944e+00,   3.05354259e+00,
+             4.56200631e+00,   5.73821643e-01,  -3.58497076e+00,
+             1.06364893e+00,  -3.73620380e+00,  -5.75226421e+00,
+            -5.45315728e+00,   1.52856248e+00,   2.14638316e+00,
+            -5.09117863e+00,   4.64406227e+00,   4.58439198e+00,
+             2.72781649e+00,  -2.64468506e+00,   2.41174739e+00])
+
+    def test_dense(self):
+        np.random.seed(42)
+        self.model.fit(self.X, self.y)
+        self.assertTrue(np.allclose(self.model.thetas, self.thetas))
+
+    def test_sparse(self):
+        X = sp.csr_matrix(self.X)
+        np.random.seed(42)
+        self.model.fit(X, self.y)
+        self.assertTrue(np.allclose(self.model.thetas, self.thetas))
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.