Matija Polajnar avatar Matija Polajnar committed e96a42a

Move reliability regression tests to add-on.

Comments (0)

Files changed (4)

docs/rst/code/reliability-basic.py

+# Description: Reliability estimation - basic & fast
+# Category:    evaluation
+# Uses:        housing
+# Referenced:  Orange.evaluation.reliability
+# Classes:     Orange.evaluation.reliability.Mahalanobis, Orange.evaluation.reliability.LocalCrossValidation, Orange.evaluation.reliability.Learner
+
+import Orange
+
+housing = Orange.data.Table("housing.tab")
+
+knn = Orange.classification.knn.kNNLearner()
+
+estimators = [Orange.evaluation.reliability.Mahalanobis(k=3),
+              Orange.evaluation.reliability.LocalCrossValidation(k = 10)]
+
+reliability = Orange.evaluation.reliability.Learner(knn, estimators = estimators)
+
+restimator = reliability(housing)
+instance = housing[0]
+
+value, probability = restimator(instance, result_type=Orange.classification.Classifier.GetBoth)
+
+for estimate in probability.reliability_estimate:
+    print estimate.method_name, estimate.estimate

docs/rst/code/reliability-long.py

+# Description: Reliability estimation
+# Category:    evaluation
+# Uses:        prostate
+# Referenced:  Orange.evaluation.reliability
+# Classes:     Orange.evaluation.reliability.Learner
+
+import Orange
+Orange.evaluation.reliability.select_with_repeat.random_generator = None
+Orange.evaluation.reliability.select_with_repeat.randseed = 42
+
+import Orange
+prostate = Orange.data.Table("prostate.tab")
+
+knn = Orange.classification.knn.kNNLearner()
+reliability = Orange.evaluation.reliability.Learner(knn)
+
+res = Orange.evaluation.testing.cross_validation([reliability], prostate)
+
+reliability_res = Orange.evaluation.reliability.get_pearson_r(res)
+
+print
+print "Estimate               r       p"
+for estimate in reliability_res:
+    print "%-20s %7.3f %7.3f" % (Orange.evaluation.reliability.METHOD_NAME[estimate[3]],
+                                 estimate[0], estimate[1])
+
+reliability = Orange.evaluation.reliability.Learner(knn, estimators=[Orange.evaluation.reliability.SensitivityAnalysis()])
+
+res = Orange.evaluation.testing.cross_validation([reliability], prostate)
+
+reliability_res = Orange.evaluation.reliability.get_pearson_r(res)
+
+print
+print "Estimate               r       p"
+for estimate in reliability_res:
+    print "%-20s %7.3f %7.3f" % (Orange.evaluation.reliability.METHOD_NAME[estimate[3]],
+                                 estimate[0], estimate[1])
+
+indices = Orange.data.sample.SubsetIndices2(prostate, p0=0.7)
+train = prostate.select(indices, 0)
+test = prostate.select(indices, 1)
+
+reliability = Orange.evaluation.reliability.Learner(knn, icv=True)
+res = Orange.evaluation.testing.learn_and_test_on_test_data([reliability], train, test)
+
+print
+print "Method used in internal cross-validation: ", Orange.evaluation.reliability.METHOD_NAME[res.results[0].probabilities[0].reliability_estimate[0].method]
+
+top5 = sorted((abs(result.probabilities[0].reliability_estimate[0].estimate), id) for id, result in enumerate(res.results))[:5]
+for estimate, id in top5:
+    print "%7.3f %i" % (estimate, id)

docs/rst/code/reliability-run.py

+# Description: Reliability estimation with cross-validation
+# Category:    evaluation
+# Uses:        housing
+# Referenced:  Orange.evaluation.reliability
+# Classes:     Orange.evaluation.reliability.Learner
+
+import Orange
+Orange.evaluation.reliability.select_with_repeat.random_generator = None
+Orange.evaluation.reliability.select_with_repeat.randseed = 42
+
+import Orange
+housing = Orange.data.Table("housing.tab")
+
+knn = Orange.classification.knn.kNNLearner()
+reliability = Orange.evaluation.reliability.Learner(knn)
+
+results = Orange.evaluation.testing.cross_validation([reliability], housing)
+
+for i, instance in enumerate(results.results[:10]):
+    print "Instance", i
+    for estimate in instance.probabilities[0].reliability_estimate:
+        print "  ", estimate.method_name, estimate.estimate

docs/rst/code/reliability_basic.py

+import Orange
+
+housing = Orange.data.Table("housing.tab")
+
+knn = Orange.classification.knn.kNNLearner()
+
+estimators = [Orange.evaluation.reliability.Mahalanobis(k=3),
+              Orange.evaluation.reliability.LocalCrossValidation(k = 10)]
+
+reliability = Orange.evaluation.reliability.Learner(knn, estimators = estimators)
+
+restimator = reliability(housing)
+instance = housing[0]
+
+value, probability = restimator(instance, result_type=Orange.classification.Classifier.GetBoth)
+
+for estimate in probability.reliability_estimate:
+    print estimate.method_name, estimate.estimate
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.