Jure Žbontar avatar Jure Žbontar committed 3de707c

initial commit

Comments (0)

Files changed (1)

+import theano
+import theano.tensor as T
+
+import numpy as np
+
+import Orange.classification
+
+
+def rectified_linear(x):
+    return T.maximum(0.0, x)
+
+class NeuralNetwork:
+    def __init__(self, input, scale, dropout=None):
+        self.output = self.output_test = input
+        self.scale = scale
+        self.srng = T.shared_randomstreams.RandomStreams(seed=42)
+        self.params = []
+        self.params_init = []
+        self.L2 = 0
+
+        if dropout is not None:
+            self.output *= self.srng.binomial(p=dropout, size=self.output.shape)
+            self.output_test *= dropout
+
+    def full(self, n_in, n_out, dropout, activation):
+        W_init = np.random.normal(scale=self.scale, size=(n_in, n_out))
+        b_init = np.zeros(n_out)
+        self.params_init.extend([W_init, b_init])
+
+        W = theano.shared(W_init, borrow=True)
+        b = theano.shared(b_init, borrow=True)
+        self.params.extend([W, b])
+
+        self.L2 += (W**2).sum()
+
+        self.output = activation(self.output.dot(W) + b)
+        self.output_test = activation(self.output_test.dot(W) + b)
+        if dropout is not None:
+            self.output *= self.srng.binomial(p=dropout, size=self.output.shape)
+            self.output_test *= dropout
+
+class MLPLearner(Orange.classification.Fitter):
+    def __init__(self, layers, dropout, L2_reg, learning_rate, iterations, scale, batch_size):
+        self.supports_multiclass = True
+        self.iterations = iterations
+        self.batch_size = batch_size
+
+        x = T.matrix()
+        y = T.matrix()
+
+        self.model = NeuralNetwork(input=x, scale=scale, dropout=dropout[0])
+
+        for prev, next, drop in zip(layers, layers[1:], dropout[1:]):
+            self.model.full(prev, next, drop, T.nnet.sigmoid)
+
+        out_clipped = T.clip(self.model.output, 1e-15, 1 - 1e-15)
+        cost = T.mean(T.nnet.binary_crossentropy(out_clipped, y)) + L2_reg * self.model.L2 / x.shape[0]
+
+        updates = []
+        for p in self.model.params:
+            updates.append((p, p - learning_rate * T.grad(cost, p)))
+        self.train_model = theano.function(inputs=[x, y], updates=updates)
+
+    def fit(self, X_tr, y_tr, W):
+        # reset params
+        for p, v in zip(self.model.params, self.model.params_init):
+            p.set_value(v)
+
+        epoch = 0
+        while epoch < self.iterations:
+            epoch += 1
+            for i in range(0, X_tr.shape[0] - self.batch_size + 1, self.batch_size):
+                self.train_model(X_tr[i:i + self.batch_size], y_tr[i:i + self.batch_size])
+        return MLPClassifier(self.model)
+
+
+class MLPClassifier(Orange.classification.Model):
+    def __init__(self, model):
+        self.model = model
+        x = T.matrix()
+        self.get_output = theano.function(inputs=[x], outputs=self.model.output_test.flatten())
+
+    def predict(self, X_te):
+        return self.get_output(X_te)
+
+
+if __name__ == '__main__':
+    d = Orange.data.Table('test-pls')
+
+    l = MLPLearner(
+        layers=[d.X.shape[1], 20, d.Y.shape[1]],
+        dropout=[0.8, 0.5, None],
+        L2_reg=0.0001,
+        learning_rate=0.1,
+        iterations=100,
+        scale=0.1,
+        batch_size=10)
+    cls = l(d)
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.