Jure Žbontar avatar Jure Žbontar committed 9bb4a7d

test cv

Comments (0)

Files changed (1)

 
 class sklearnLearner(Orange.classification.Fitter):
     def __init__(self, model):
-        self.supports_multiclass = True
+        #self.supports_multiclass = True
         self.model = model
 
     def fit(self, X, Y, W):
-        self.model.fit(X, Y)
+        self.model.fit(X, Y.ravel())
         return sklearnModel(self.model)
 
 class sklearnModel(Orange.classification.Model):
         self.model = model
 
     def predict(self, X):
-        return self.model.predict(X)
+        return self.model.predict_proba(X)
 
 
 # Multilayer perceptron
         self.stacker_cls = stacker_cls
 
     def predict(self, X):
-        print(self.cls(X))
-        return self.cls(X)
+        return self.stacker_cls.predict(self.cls.predict(X))
 
 
 
 
 
 
-    d = Orange.data.Table('test-pls')
     #d.X = (d.X - np.mean(d.X, axis=0)) / np.std(d.X, axis=0)
 
     #d = np.loadtxt('/home/jure/tmp/water-quality-nom.arff', skiprows=57, delimiter=',')
 #    sys.exit()
 
 
-    model = MTStackLearner(
-        model=BRsklearnLearner(sklearn.linear_model.Ridge(alpha=1)),
-        stacker=BRsklearnLearner(sklearn.linear_model.Ridge(alpha=1)),
-    )
-    print(np.mean(cross_val_score(model, d)))
-
-    model = MeanRegressorLearner()
-    print(np.mean(cross_val_score(model, d)))
-
-    model = sklearnLearner(sklearn.cross_decomposition.PLSRegression(n_components=2))
-    print(np.mean(cross_val_score(model, d)))
-
-    #model = BRsklearnLearner(sklearn.ensemble.RandomForestRegressor(n_estimators=100))
+    #d = Orange.data.Table('test-pls')
+    #model = MTStackLearner(
+    #    model=BRsklearnLearner(sklearn.linear_model.Ridge(alpha=1)),
+    #    stacker=BRsklearnLearner(sklearn.linear_model.Ridge(alpha=1)),
+    #)
     #print(np.mean(cross_val_score(model, d)))
 
-    model = BRsklearnLearner(sklearn.linear_model.LinearRegression())
-    print(np.mean(cross_val_score(model, d)))
+    import ml_metrics
+    import Orange.evaluation
+    import sklearn.metrics
 
-    model = BRsklearnLearner(sklearn.linear_model.Ridge(alpha=0.1))
-    print(np.mean(cross_val_score(model, d)))
 
-    model = CurdsWheyRegressorLearner('population')
-    print(np.mean(cross_val_score(model, d)))
+    d = Orange.data.Table('monks-1')
+
+    model = sklearnLearner(sklearn.linear_model.LogisticRegression())
+    cv = Orange.evaluation.CrossValidation(d, model)
+    p = cv.KFold(5)[1]
+    #print(p)
+    print("orange    ", Orange.evaluation.AUC_binary(d, p))
+    print("ml_metrics", ml_metrics.auc(d.Y.ravel(), p[:,1]))
+    print("sklearn   ", sklearn.metrics.roc_auc_score(d.Y.ravel(), p[:,1]))
+
+
+
+
+    #X, y = d.X, d.Y.ravel()
+    #p = np.zeros_like(y)
+    #model = sklearn.linear_model.LogisticRegression()
+    #
+    #for tr, te in sklearn.cross_validation.KFold(X.shape[0], 5):
+    #    model.fit(X[tr], y[tr])
+    #    p[te] = model.predict_proba(X[te])[:,1]
+    #print(ml_metrics.auc(y, p))
+
+
+
+
+
+    #model = MeanRegressorLearner()
+    #print(np.mean(cross_val_score(model, d)))
+    #
+    #model = sklearnLearner(sklearn.cross_decomposition.PLSRegression(n_components=2))
+    #print(np.mean(cross_val_score(model, d)))
+    #
+    ##model = BRsklearnLearner(sklearn.ensemble.RandomForestRegressor(n_estimators=100))
+    ##print(np.mean(cross_val_score(model, d)))
+    #
+    #model = BRsklearnLearner(sklearn.linear_model.LinearRegression())
+    #print(np.mean(cross_val_score(model, d)))
+    #
+    #model = BRsklearnLearner(sklearn.linear_model.Ridge(alpha=0.1))
+    #print(np.mean(cross_val_score(model, d)))
+    #
+    #model = CurdsWheyRegressorLearner('population')
+    #print(np.mean(cross_val_score(model, d)))
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.