Commits

Miran Levar committed c1791dc

Added multitarget learners and classifiers.

Comments (0)

Files changed (7)

 
 .. _Orange: http://orange.biolab.si/
 
-Documentation is found at:
+Documentation will be found at:
 
 http://orange-multitarget.readthedocs.org/
+
+
+TODO:
+write the missing documentation (replace the placeholders)
+add datasets

_multitarget/__init__.py

-foo=42
+"""
+Wrapper for constructing multi-target learners
+==============================================
+
+This module also contains a wrapper, an auxilary learner, that can be used
+to construct simple multi-target learners from standard learners designed
+for data with a single class. The wrapper uses the specified base learner
+to construct independent models for each class.
+
+.. index:: MultitargetLearner
+.. autoclass:: Orange.multitarget.MultitargetLearner
+    :members:
+    :show-inheritance:
+
+.. index:: MultitargetClassifier
+.. autoclass:: Orange.multitarget.MultitargetClassifier
+    :members:
+    :show-inheritance:
+
+Examples
+========
+
+The following example uses a simple multi-target data set (generated with
+:download:`generate_multitarget.py <code/generate_multitarget.py>`) to show
+some basic functionalities (part of
+:download:`multitarget.py <code/multitarget.py>`).
+
+.. literalinclude:: code/multitarget.py
+    :lines: 1-6
+
+Multi-target learners can build prediction models (classifiers)
+which then predict (multiple) class values for a new instance (continuation of
+:download:`multitarget.py <code/multitarget.py>`):
+
+.. literalinclude:: code/multitarget.py
+    :lines: 8-
+
+"""
+
+import Orange
+
+# Other algorithms which also work with multitarget data
+from Orange.regression import pls
+from Orange.regression import earth
+
+
+class MultitargetLearner(Orange.classification.Learner):
+    """
+    Wrapper for multitarget problems that constructs independent models
+    of a base learner for each class variable.
+
+    .. attribute:: learner
+
+        The base learner used to learn models for each class.
+    """
+
+    def __new__(cls, learner, data=None, weight=0, **kwargs):
+        self = Orange.classification.Learner.__new__(cls, **kwargs)
+        if data:
+            self.__init__(learner, **kwargs)
+            return self.__call__(data, weight)
+        else:
+            return self
+    
+    def __init__(self, learner, **kwargs):
+        """
+
+        :param learner: Base learner used to construct independent
+                        models for each class.
+        """
+
+        self.learner = learner
+        self.__dict__.update(kwargs)
+
+    def __call__(self, data, weight=0):
+        """
+        Learn independent models of the base learner for each class.
+
+        :param data: Multitarget data instances (with more than 1 class).
+        :type data: :class:`Orange.data.Table`
+
+        :param weight: Id of meta attribute with weights of instances
+        :type weight: :obj:`int`
+
+        :rtype: :class:`Orange.multitarget.MultitargetClassifier`
+        """
+
+        if not data.domain.class_vars:
+            raise Exception('No classes defined.')
+        
+        domains = [Orange.data.Domain(data.domain.attributes, y)
+                   for y in data.domain.class_vars]
+        classifiers = [self.learner(Orange.data.Table(dom, data), weight)
+                       for dom in domains]
+        return MultitargetClassifier(classifiers=classifiers, domains=domains)
+        
+    def __reduce__(self):
+        return type(self), (self.learner,), dict(self.__dict__)
+
+
+class MultitargetClassifier(Orange.classification.Classifier):
+    """
+    Multitarget classifier that returns a list of predictions from each
+    of the independent base classifiers.
+
+    .. attribute classifiers
+
+        List of individual classifiers for each class.
+    """
+
+    def __init__(self, classifiers, domains):
+        self.classifiers = classifiers
+        self.domains = domains
+
+    def __call__(self, instance, return_type=Orange.core.GetValue):
+        """
+        :param instance: Instance to be classified.
+        :type instance: :class:`Orange.data.Instance`
+
+        :param return_type: One of
+            :class:`Orange.classification.Classifier.GetValue`,
+            :class:`Orange.classification.Classifier.GetProbabilities` or
+            :class:`Orange.classification.Classifier.GetBoth`
+        """
+
+        predictions = [c(Orange.data.Instance(dom, instance), return_type)
+                       for c, dom in zip(self.classifiers, self.domains)]
+        return zip(*predictions) if return_type == Orange.core.GetBoth \
+               else predictions
+
+    def __reduce__(self):
+        return type(self), (self.classifiers, self.domains), dict(self.__dict__)
+

_multitarget/binary.py

+import Orange.core as orange
+import Orange
+import random
+import copy
+from operator import add
+
+class BinaryRelevanceLearner(orange.Learner):
+    """
+    Expands single class classification techniques into multi-target classification techniques by chaining the classification
+    data. A learner is constructed for each of the class variables in a random or given order. The data for each learner are
+    the features extended by all previously classified variables. This chaining passes the class informationd between
+    classifiers and allows class correlations to be taken into account.
+    TODO: cite weka source?
+
+    :param learner: A single class learner that will be extended by chaining.
+    :type learner: :class:`Orange.core.Learner`
+
+    :param rand: random generator used in bootstrap sampling. If None (default), 
+        then ``random.Random(42)`` is used.
+
+    :param callback: a function to be called after every iteration of
+            induction of classifier. The call returns a parameter
+            (from 0.0 to 1.0) that provides an estimate
+            of completion of the learning progress.
+
+    :param name: learner name.
+    :type name: string
+
+    :rtype: :class:`Orange.ensemble.forest.RandomForestClassifier` or 
+            :class:`Orange.ensemble.forest.RandomForestLearner`
+
+    """
+
+    def __new__(cls, data=None, weight = 0, **kwargs):
+        self = Orange.classification.Learner.__new__(cls, **kwargs)
+        if data:   
+            self.__init__(**kwargs)
+            return self(data,weight)
+        else:
+            return self
+
+    def __init__(self, learner=None, name="Binary Relevance", rand=None, callback=None):
+        self.name = name
+        self.rand = rand
+        self.callback = callback
+
+        if not learner:
+            raise TypeError("Wrong specification, learner not defined")
+        else:
+            self.learner = learner
+
+        if not self.rand:
+            self.rand = random.Random(42)
+
+        self.randstate = self.rand.getstate()
+           
+
+    def __call__(self, instances, weight=0):
+        """
+        Learn from the given table of data instances.
+        
+        :param instances: data for learning.
+        :type instances: class:`Orange.data.Table`
+
+        :param weight: weight.
+        :type weight: int
+
+        :rtype: :class:`Orange.ensemble.chain.ClassifierChain`
+        """
+
+        instances = Orange.data.Table(instances.domain, instances) # bypasses ownership
+
+        self.rand.setstate(self.randstate) 
+        n = len(instances)
+        m = len(instances.domain.class_vars)
+        progress = 0.0
+
+        classifiers = [None for _ in xrange(m)]
+        domains = [None for _ in xrange(m)]
+        orig_domain = copy.copy(instances.domain)
+
+        class_order = [cv for cv in instances.domain.class_vars]
+
+        learner = self.learner
+
+        for i in range(m):
+            # sets one of the class_vars as class_var
+            instances.pick_class(class_order[i])            
+
+            # save domains for classification
+            domains[i] = Orange.data.Domain([d for d in instances.domain])
+
+            classifiers[i] = learner(instances, weight)
+
+            if self.callback:
+                progress+=1
+                self.callback(progress / m)
+
+        return BinaryRelevanceClassifier(classifiers=classifiers, class_order=class_order, domains=domains, name=self.name, orig_domain=orig_domain)
+
+
+class BinaryRelevanceClassifier(orange.Classifier):
+    """
+    Uses the classifiers induced by the :obj:`ClassifierChainLearner`. An input
+    instance is classified into the class with the most frequent vote.
+    However, this implementation returns the averaged probabilities from
+    each of the trees if class probability is requested.
+
+    It should not be constructed manually. TODO: ask about this
+    
+    :param classifiers: a list of classifiers to be used.
+    :type classifiers: list
+    
+    
+    
+    :param domains: the domain of the learning set.
+    :type domain: :class:`Orange.data.Domain`
+    
+    :param class_var: the class feature.
+    :type class_var: :class:`Orange.feature.Descriptor`
+
+    :param class_vars: the multi-target class features.
+    :type class_vars: list of :class:`Orange.feature.Descriptor`
+
+    :param name: name of the resulting classifier.
+    :type name: str
+
+    """
+
+    def __init__(self, classifiers, class_order, domains, name, orig_domain):
+        self.classifiers = classifiers
+        self.class_order = class_order
+        self.name = name
+        self.domains = domains
+        self.orig_domain = orig_domain
+
+    def __call__(self, instance, result_type = orange.GetValue):
+        """
+        :param instance: instance to be classified.
+        :type instance: :class:`Orange.data.Instance`
+        
+        :param result_type: :class:`Orange.classification.Classifier.GetValue` or \
+              :class:`Orange.classification.Classifier.GetProbabilities` or
+              :class:`Orange.classification.Classifier.GetBoth`
+        
+        :rtype: :class:`Orange.data.Value`, 
+              :class:`Orange.statistics.Distribution` or a tuple with both
+        """
+        m = len(self.class_order)
+        values = [None for _ in range(m)] 
+        probs = [None for _ in range(m)] 
+
+        for i in range(m):
+            #add blank class for classification
+            inst = Orange.data.Instance(self.domains[i], [v for v in instance]+['?'])
+
+            res = self.classifiers[i](inst, orange.GetBoth)
+            values[i] = res[0]
+            probs[i] = res[1]
+
+        if result_type == orange.GetValue: return tuple(values)
+        elif result_type == orange.GetProbabilities: return tuple(probs)
+        else: 
+            return [tuple(values),tuple(probs)]
+
+    def __reduce__(self):
+        return type(self), (self.classifiers, self.class_order, self.domains, self.name, self.orig_domain), dict(self.__dict__)
+
+if __name__ == '__main__':
+    import time
+    print "STARTED"
+    global_timer = time.time()
+
+    data = Orange.data.Table('bridges.v2.nm')
+    #data = Orange.data.Table('ntp.fp3.nm')
+    
+    
+    l1 = BinaryRelevanceLearner(learner = Orange.classification.tree.SimpleTreeLearner)
+    l2 = BinaryRelevanceLearner(learner = Orange.classification.bayes.NaiveLearner)
+    l3 = BinaryRelevanceLearner(learner = Orange.classification.majority.MajorityLearner)
+    l4 = Orange.multitarget.tree.MultiTreeLearner()
+
+
+    #l =  EnsembleClassifierChainLearner(learner = Orange.classification.tree.SimpleTreeLearner, n_chains=3, name="ECC T")
+    #l = Orange.multitarget.tree.MultiTreeLearner()
+    #l = EnsembleClassifierChainLearner(learner = Orange.classification.majority.MajorityLearner, n_chains=3, name="ECC M")
+    # cross_validation not working
+    res = Orange.evaluation.testing.cross_validation([l1,l2,l3,l4],data, folds = 2)
+    #res = Orange.evaluation.testing.cross_validation([l],data,folds=5)
+
+    scores = Orange.evaluation.scoring.mt_average_score(res,Orange.evaluation.scoring.RMSE)
+   
+    print scores
+    print res.classifierNames
+    for i in range(len(scores)):
+        print res.classifierNames[i], scores[i]
+
+
+    print "--DONE %.2f --" % (time.time()-global_timer)

_multitarget/chain.py

+import Orange.core as orange
+import Orange
+import random
+import copy
+from operator import add
+
+class ClassifierChainLearner(orange.Learner):
+    """
+    Expands single class classification techniques into multi-target classification techniques by chaining the classification
+    data. A learner is constructed for each of the class variables in a random or given order. The data for each learner are
+    the features extended by all previously classified variables. This chaining passes the class informationd between
+    classifiers and allows class correlations to be taken into account.
+    TODO: cite weka source?
+
+    :param learner: A single class learner that will be extended by chaining.
+    :type learner: :class:`Orange.core.Learner`
+
+    :param rand: random generator used in bootstrap sampling. If None (default), 
+        then ``random.Random(42)`` is used.
+
+    :param callback: a function to be called after every iteration of
+            induction of classifier. The call returns a parameter
+            (from 0.0 to 1.0) that provides an estimate
+            of completion of the learning progress.
+
+    :param name: learner name.
+    :type name: string
+
+    :rtype: :class:`Orange.ensemble.forest.RandomForestClassifier` or 
+            :class:`Orange.ensemble.forest.RandomForestLearner`
+
+    """
+
+    def __new__(cls, data=None, weight = 0, **kwargs):
+        self = Orange.classification.Learner.__new__(cls, **kwargs)
+        if data:   
+            self.__init__(**kwargs)
+            return self(data,weight)
+        else:
+            return self
+
+    def __init__(self, learner=None, name="Classifier Chain", rand=None, callback=None, class_order=None):
+        self.name = name
+        self.rand = rand
+        self.callback = callback
+        self.class_order = class_order
+
+        if not learner:
+            raise TypeError("Wrong specification, learner not defined")
+        else:
+            self.learner = learner
+
+        if not self.rand:
+            self.rand = random.Random(42)
+
+        self.randstate = self.rand.getstate()
+           
+
+    def __call__(self, instances, weight=0, class_order=None):
+        """
+        Learn from the given table of data instances.
+        
+        :param instances: data for learning.
+        :type instances: class:`Orange.data.Table`
+
+        :param weight: weight.
+        :type weight: int
+
+        :param class_order: list of descriptors of class variables
+        :type class_order: list of :class:`Orange.feature.Descriptor`
+
+        :rtype: :class:`Orange.ensemble.chain.ClassifierChain`
+        """
+
+        instances = Orange.data.Table(instances.domain, instances) # bypasses ownership
+
+        self.rand.setstate(self.randstate) 
+        n = len(instances)
+        m = len(instances.domain.class_vars)
+        progress = 0.0
+
+        classifiers = [None for _ in xrange(m)]
+        domains = [None for _ in xrange(m)]
+        orig_domain = copy.copy(instances.domain)
+
+        if self.class_order and not class_order:
+            class_order = self.class_order
+        elif not class_order:
+            class_order = [cv for cv in instances.domain.class_vars]
+            self.rand.shuffle(class_order)
+        print [cv.name for cv in class_order]
+        learner = self.learner
+
+        for i in range(m):
+            # sets one of the class_vars as class_var
+            instances.pick_class(class_order[i])            
+
+            # save domains for classification
+            domains[i] = Orange.data.Domain([d for d in instances.domain])
+
+            classifiers[i] = learner(instances, weight)
+            
+            # updates domain to include class_var in features
+            instances.change_domain(Orange.data.Domain(instances.domain, False, \
+                class_vars=instances.domain.class_vars))
+
+            if self.callback:
+                progress+=1
+                self.callback(progress / m)
+
+        return ClassifierChain(classifiers=classifiers, class_order=class_order, domains=domains, name=self.name, orig_domain=orig_domain)
+
+
+class ClassifierChain(orange.Classifier):
+    """
+    Uses the classifiers induced by the :obj:`ClassifierChainLearner`. An input
+    instance is classified into the class with the most frequent vote.
+    However, this implementation returns the averaged probabilities from
+    each of the trees if class probability is requested.
+
+    It should not be constructed manually. TODO: ask about this
+    
+    :param classifiers: a list of classifiers to be used.
+    :type classifiers: list
+    
+    
+    
+    :param domains: the domain of the learning set.
+    :type domain: :class:`Orange.data.Domain`
+    
+    :param class_var: the class feature.
+    :type class_var: :class:`Orange.feature.Descriptor`
+
+    :param class_vars: the multi-target class features.
+    :type class_vars: list of :class:`Orange.feature.Descriptor`
+
+    :param name: name of the resulting classifier.
+    :type name: str
+
+    """
+
+    def __init__(self, classifiers, class_order, domains, name, orig_domain):
+        self.classifiers = classifiers
+        self.class_order = class_order
+        self.name = name
+        self.domains = domains
+        self.orig_domain = orig_domain
+
+    def __call__(self, instance, result_type = orange.GetValue):
+        """
+        :param instance: instance to be classified.
+        :type instance: :class:`Orange.data.Instance`
+        
+        :param result_type: :class:`Orange.classification.Classifier.GetValue` or \
+              :class:`Orange.classification.Classifier.GetProbabilities` or
+              :class:`Orange.classification.Classifier.GetBoth`
+        
+        :rtype: :class:`Orange.data.Value`, 
+              :class:`Orange.statistics.Distribution` or a tuple with both
+        """
+
+        inst = [v for v in instance]
+        values = {cv:None for cv in instance.domain.class_vars}
+        probs = {cv:None for cv in instance.domain.class_vars}
+
+
+
+        for i in range(len(self.class_order)):
+            # add blank class for classification
+
+            inst = Orange.data.Instance(self.domains[i],[v for v in inst] + ['?'])
+  
+            res = self.classifiers[i](inst, orange.GetBoth)
+            values[self.class_order[i]] = res[0]
+            probs[self.class_order[i]] = res[1]
+            inst[-1] = res[0] # set the classified value for chaining
+
+        # sort values and probabilites to original order
+        values = [values[cv] for cv in self.orig_domain.class_vars]
+        probs = [probs[cv] for cv in self.orig_domain.class_vars]
+
+        if result_type == orange.GetValue: return tuple(values)
+        elif result_type == orange.GetProbabilities: return tuple(probs)
+        else: 
+            return [tuple(values),tuple(probs)]
+
+    def __reduce__(self):
+        return type(self), (self.classifiers, self.class_order, self.domains, self.name, self.orig_domain), dict(self.__dict__)
+
+
+class EnsembleClassifierChainLearner(orange.Learner):
+    """
+    Expands single class classification techniques into multi-target classification techniques by chaining the classification
+    data. A learner is constructed for each of the class variables in a random or given order. The data for each learner are
+    the features extended by all previously classified variables. This chaining passes the class informationd between
+    classifiers and allows class correlations to be taken into account.
+    TODO: cite weka source?
+
+    :param learner: A single class learner that will be extended by chaining.
+    :type learner: :class:`Orange.core.Learner`
+
+    :param rand: random generator used in bootstrap sampling. If None (default), 
+        then ``random.Random(42)`` is used.
+
+    :param callback: a function to be called after every iteration of
+            induction of classifier. The call returns a parameter
+            (from 0.0 to 1.0) that provides an estimate
+            of completion of the learning progress.
+
+    :param name: learner name.
+    :type name: string
+
+    :rtype: :class:`Orange.ensemble.forest.RandomForestClassifier` or 
+            :class:`Orange.ensemble.forest.RandomForestLearner`
+    """
+
+    def __new__(cls, data=None, weight = 0, **kwargs):
+        self = Orange.classification.Learner.__new__(cls, **kwargs)
+        if data:   
+            self.__init__(**kwargs)
+            return self(data,weight)
+        else:
+            return self
+
+    def __init__(self, n_chains=50, sample_size=0.25, learner=None, name="Ensemble CChain", rand=None, callback=None):
+        self.n_chains = n_chains
+        self.sample_size = sample_size
+        self.name = name
+        self.rand = rand
+        self.callback = callback
+
+        if not learner:
+            raise TypeError("Wrong specification, learner not defined")
+        else:
+            self.learner = learner
+
+        if not self.rand:
+            self.rand = random.Random(42)
+
+        self.randstate = self.rand.getstate()
+
+    def __call__(self, instances, weight=0):
+        """
+        """
+
+        ind = Orange.data.sample.SubsetIndices2(p0=1-self.sample_size)
+        indices = ind(instances)
+        class_order = [cv for cv in instances.domain.class_vars]
+        classifiers = []
+
+        for i in range(self.n_chains):
+            #randomize chain ordering and subset of instances
+            self.rand.shuffle(class_order)
+            self.rand.shuffle(indices)
+            data = instances.select_ref(indices,1)
+
+            learner = ClassifierChainLearner(learner = self.learner, rand=self.rand) # TODO might work in one step
+
+            classifiers.append(learner(data, weight, copy.copy(class_order)))
+
+            if self.callback:
+                self.callback((i+1.)/self.n_chains)
+
+        return EnsembleClassifierChain(classifiers=classifiers, name=self.name)
+
+class EnsembleClassifierChain(orange.Classifier):
+    """
+    Uses the classifiers induced by the :obj:`EnsembleClassifierChainLearner`. An input
+    instance is classified into the class with the most frequent vote.
+    However, this implementation returns the averaged probabilities from
+    each of the trees if class probability is requested.
+
+    It should not be constructed manually. TODO: ask about this
+    
+    When constructed manually, the following parameters have to
+    be passed:
+
+    :param classifiers: a list of classifiers to be used.
+    :type classifiers: list
+    
+    
+    
+    :param domains: the domain of the learning set.
+    :type domain: :class:`Orange.data.Domain`
+    
+    :param class_var: the class feature.
+    :type class_var: :class:`Orange.feature.Descriptor`
+
+    :param class_vars: the multi-target class features.
+    :type class_vars: list of :class:`Orange.feature.Descriptor`
+
+    :param name: name of the resulting classifier.
+    :type name: str
+
+    """
+
+    def __init__(self, classifiers, name):
+        self.classifiers = classifiers
+        self.name = name
+
+    def __call__(self, instance, result_type = orange.GetValue):
+        """
+        :param instance: instance to be classified.
+        :type instance: :class:`Orange.data.Instance`
+        
+        :param result_type: :class:`Orange.classification.Classifier.GetValue` or \
+              :class:`Orange.classification.Classifier.GetProbabilities` or
+              :class:`Orange.classification.Classifier.GetBoth`
+        
+        :rtype: :class:`Orange.data.Value`, 
+              :class:`Orange.statistics.Distribution` or a tuple with both
+        """
+
+        class_vars = instance.domain.class_vars
+
+        # get results to avoid multiple calls
+        res_both = [c(instance, orange.GetBoth) for c in self.classifiers]
+
+        mt_prob = []
+        mt_value = []
+
+        for varn in xrange(len(class_vars)):
+
+            class_var = class_vars[varn]
+            # handle discreete class
+        
+            if class_var.var_type == Orange.feature.Discrete.Discrete:
+
+                # voting for class probabilities
+                if result_type == orange.GetProbabilities or result_type == orange.GetBoth:
+                    prob = [0.] * len(class_var.values)
+                    for r in res_both:
+                        a = [x for x in r[1][varn]]
+                        prob = map(add, prob, a)
+                    norm = sum(prob)
+                    cprob = Orange.statistics.distribution.Discrete(class_var)
+                    for i in range(len(prob)):
+                        cprob[i] = prob[i]/norm
+                
+                # voting for crisp class membership, notice that
+                # this may not be the same class as one obtaining the
+                # highest probability through probability voting
+                if result_type == orange.GetValue or result_type == orange.GetBoth:
+                    cfreq = [0] * len(class_var.values)
+                    for r in res_both:
+                        cfreq[int(r[0][varn])] += 1
+                    index = cfreq.index(max(cfreq))
+                    cvalue = Orange.data.Value(class_var, index)
+            
+                if result_type == orange.GetValue: mt_value.append(cvalue)
+                elif result_type == orange.GetProbabilities: mt_prob.append(cprob)
+                else: 
+                    mt_value.append(cvalue)
+                    mt_prob.append(cprob)
+        
+            else:
+                # Handle continuous class
+        
+                # voting for class probabilities
+                if result_type == orange.GetProbabilities or result_type == orange.GetBoth:
+                    probs = [ r for r in res_both]
+                    cprob = dict()
+      
+                    for val,prob in probs:
+                        if prob[varn] != None: #no probability output
+                            a = dict(prob[varn].items())
+                        else:
+                            a = { val[varn].value : 1. }
+                        cprob = dict( (n, a.get(n, 0)+cprob.get(n, 0)) for n in set(a)|set(cprob) )
+                    cprob = Orange.statistics.distribution.Continuous(cprob)
+                    cprob.normalize()
+                
+                # gather average class value
+                if result_type == orange.GetValue or result_type == orange.GetBoth:
+                    values = [r[0][varn] for r in res_both]
+                    cvalue = Orange.data.Value(class_var, sum(values) / len(self.classifiers))
+            
+                if result_type == orange.GetValue: mt_value.append(cvalue)
+                elif result_type == orange.GetProbabilities: mt_prob.append(cprob)
+                else: 
+                    mt_value.append(cvalue)
+                    mt_prob.append(cprob)
+                    
+        if result_type == orange.GetValue: return tuple(mt_value)
+        elif result_type == orange.GetProbabilities: return tuple(mt_prob)
+        else: 
+            return [tuple(mt_value),tuple(mt_prob)]
+
+    def __reduce__(self):
+        return type(self), (self.classifiers, self.class_order, self.domains, self.name), dict(self.__dict__)
+
+
+if __name__ == '__main__':
+    data = Orange.data.Table('test4')
+    l=ClassifierChainLearner(learner = Orange.classification.tree.SimpleTreeLearner, name="CC T")
+    res = Orange.evaluation.testing.cross_validation([l],data,folds=2)
+
+    exit()
+
+    print "STARTED"
+    import time
+    global_timer = time.time()
+    data = Orange.data.Table('bridges.v2.nm')
+    #data = Orange.data.Table('bridges.v2.nm')
+    cl3 = EnsembleClassifierChainLearner(learner = Orange.classification.tree.SimpleTreeLearner, n_chains=50, sample_size=0.25, name="ECC T", rand = random.seed(time.time()))
+    cl4 = EnsembleClassifierChainLearner(learner = Orange.classification.majority.MajorityLearner, n_chains=50, sample_size=0.25, name="ECC M", rand = random.seed(time.time()))
+    cl1 = ClassifierChainLearner(learner = Orange.classification.tree.SimpleTreeLearner, name="CC T")
+    cl2 = ClassifierChainLearner(learner = Orange.classification.majority.MajorityLearner, name="CC M")
+    l1 = Orange.multitarget.tree.MultiTreeLearner()
+    l2 = Orange.multitarget.pls.PLSRegressionLearner()
+    l3 = Orange.ensemble.forest.RandomForestLearner(base_learner=Orange.multitarget.tree.MultiTreeLearner(), trees=50, name="RF MT" )
+
+    #l =  EnsembleClassifierChainLearner(learner = Orange.classification.tree.SimpleTreeLearner, n_chains=3, name="ECC T")
+    #l = Orange.multitarget.tree.MultiTreeLearner()
+    #l = EnsembleClassifierChainLearner(learner = Orange.classification.majority.MajorityLearner, n_chains=3, name="ECC M")
+    # cross_validation not working
+    res = Orange.evaluation.testing.cross_validation([cl1,cl2,cl3,cl4,l1,l2,l3],data,folds=2)
+    #res = Orange.evaluation.testing.cross_validation([l],data,folds=5)
+
+    scores = Orange.evaluation.scoring.mt_average_score(res,Orange.evaluation.scoring.RMSE)
+   
+    print scores
+    print res.classifierNames
+    for i in range(len(scores)):
+        print res.classifierNames[i], scores[i]
+
+    print "CA"
+    scores = Orange.evaluation.scoring.mt_average_score(res,Orange.evaluation.scoring.CA)
+    print scores
+    print res.classifierNames
+    for i in range(len(scores)):
+        print res.classifierNames[i], scores[i]
+
+
+    #indices = Orange.data.sample.SubsetIndices2(data,0.25)
+    #data_learn = data.select(indices,1)
+    #data_test = data.select(indices,0)
+
+    #c1=cl3(data_learn)
+    #c2=l1(data_learn)
+    #
+    #res = Orange.evaluation.testing.test_on_data([c1, c2],data_test)
+    #print Orange.evaluation.scoring.mt_average_score(res,Orange.evaluation.scoring.RMSE)
+
+    print "--DONE %.2f --" % (time.time()-global_timer)

_multitarget/neural.py

+import Orange
+import numpy as np
+np.seterr('ignore') # set to ignore to disable overflow errors
+np.random.seed(42) # TODO: check with Jure
+import scipy.sparse
+from scipy.optimize import fmin_l_bfgs_b
+
+class _NeuralNetwork:
+    def __init__(self, layers,lambda_=1, callback=None, **fmin_args):
+        self.layers = layers
+        self.lambda_ = lambda_
+        self.callback = callback
+        self.fmin_args = fmin_args
+
+    def unfold_params(self, params):
+        i0, i1, i2 = self.layers
+
+        i = (i0 + 1) * i1
+
+        Theta1 = params[:i].reshape((i1, i0 + 1))
+        Theta2 = params[i:].reshape((i2, i1 + 1))
+
+        return Theta1, Theta2
+
+    def cost_grad(self, params):
+        Theta1, Theta2 = self.unfold_params(params)
+
+        if self.callback:
+            self.Theta1 = Theta1
+            self.Theta2 = Theta2
+            self.callback(self)
+
+        # Feedforward Propagation
+        m, n = self.X.shape
+
+        a1 = self.X
+        z2 = a1.dot(Theta1.T)
+        a2 = np.column_stack((np.ones(m), _sigmoid(z2)))
+        z3 = a2.dot(Theta2.T)
+        a3 = _sigmoid(z3)
+
+        # Cost
+        J = np.sum(-self.y * np.log(a3) - (1 - self.y) * np.log(1 - a3)) / m
+
+        t1 = Theta1.copy()
+        t1[:, 0] = 0
+        t2 = Theta2.copy()
+        t2[:, 0] = 0
+
+        # regularization
+        reg = np.dot(t1.flat, t1.flat)
+        reg += np.dot(t2.flat, t2.flat)
+        J += float(self.lambda_) * reg / (2.0 * m)
+
+        # Grad
+        d3 = a3 - self.y
+        d2 = d3.dot(Theta2)[:, 1:] * sigmoid_gradient(z2)
+
+        D2 = a2.T.dot(d3).T / m
+        D1 = a1.T.dot(d2).T / m
+
+        # regularization
+        D2 += t2 * (float(self.lambda_) / m)
+        D1 += t1 * (float(self.lambda_) / m)
+
+        return J, np.hstack((D1.flat, D2.flat))
+
+    def fit(self, X, y):
+        i0, i1, i2 = self.layers
+
+        m, n = X.shape
+        n_params = i1 * (i0 + 1) + i2 * (i1 + 1)
+        eps = np.sqrt(6) / np.sqrt(i0 + i2)
+        initial_params = np.random.randn(n_params) * 2 * eps - eps
+
+        self.X = self.append_ones(X)
+        self.y = y
+
+        params, _, _ = fmin_l_bfgs_b(self.cost_grad, initial_params, **self.fmin_args)
+
+        self.Theta1, self.Theta2 = self.unfold_params(params)
+
+    def predict(self, X):
+        m, n = X.shape
+        
+        a2 = _sigmoid(self.append_ones(X).dot(self.Theta1.T))
+        a3 = _sigmoid(np.column_stack((np.ones(m), a2)).dot(self.Theta2.T))
+
+        return a3
+
+    def append_ones(self, X):
+        m, n = X.shape
+        if scipy.sparse.issparse(X):
+            return scipy.sparse.hstack((np.ones((m, 1)), X)).tocsr()
+        else:
+            return np.column_stack((np.ones(m), X))
+
+def _sigmoid(x):
+    return 1.0 / (1.0 + np.exp(-x))
+
+def _sigmoid_gradient(x):
+    sx = _sigmoid(x)
+    return sx * (1 - sx)
+
+
+class NeuralNetworkLearner(Orange.classification.Learner):
+    """
+    NeuralNetworkLearner uses jzbontar's implementation of neural networks and wraps it in
+    an Orange compatible learner.
+    """
+
+    def __new__(cls, data=None, weight = 0, **kwargs):
+        self = Orange.classification.Learner.__new__(cls, **kwargs)
+
+        if data is None:   
+            return self
+        else:
+            self.__init__(**kwargs)
+            return self(data,weight)
+
+    def __init__(self, name="NeuralNetwork",n_mid=10,reg_fact=1,max_iter=10,treshold=0.5):
+        """
+        Current default values are the same as in the original implementation (neural_networks.py)
+        Currently supports only multi-label data.
+        """
+        self.name = name
+        self.n_mid = n_mid
+        self.reg_fact=reg_fact
+        self.max_iter=max_iter
+        self.treshold = treshold
+
+    def __call__(self,data,weight=0):
+        
+        #converts attribute data
+        X = data.to_numpy()[0] 
+
+        #converts multitarget classes
+        Y = np.array([[float(c) for c in d.get_classes()] for d in data])
+
+        #initializes neural networks
+        self.nn =  _NeuralNetwork([len(X[0]),self.n_mid,len(Y[0])], lambda_=self.reg_fact,maxfun=self.max_iter, iprint=-1)
+        
+        self.nn.fit(X,Y)
+               
+        return NeuralNetworkClassifier(classifier=self.nn.predict, domain = data.domain, treshold=self.treshold)
+
+
+class NeuralNetworkClassifier():
+    """Classifier returned by NeuralNetworkLearner"""
+    def __init__(self,**kwargs):
+        self.__dict__.update(**kwargs)
+
+    def __call__(self,example, result_type=Orange.core.GetValue):
+        # transform example to numpy
+        input = numpy.array([[float(e) for e in example]])
+        # transform results from numpy
+        results = self.classifier(input).tolist()[0]
+        mt_prob = []
+        mt_value = []
+        
+        for varn in range(len(self.domain.class_vars)):
+            # handle discrete class
+
+            if self.domain.class_vars[varn].var_type == Orange.feature.Discrete.Discrete:
+
+                cprob = Orange.statistics.distribution.Discrete(self.domain.class_vars[varn])
+                cprob[0] = 1.-results[varn]
+                cprob[1] = float(results[varn])
+                mt_prob.append(cprob)
+                mt_value.append(Orange.data.Value(Orange.feature.Continuous(self.domain.class_vars[varn].name),results[varn]))
+            else:
+                raise ValueError("non-discrete classes not supported")
+        
+        if result_type == orange.GetValue: return tuple(mt_value)
+        elif result_type == orange.GetProbabilities: return tuple(mt_prob)
+        else: 
+            return [tuple(mt_value),tuple(mt_prob)]
+"""
+.. index:: Multi-target Tree Learner
+
+***************************************
+Multi-target Tree Learner
+***************************************
+
+To use the tree learning algorithm for multi-target data, standard
+orange trees (:class:`Orange.classification.tree.TreeLearner`) can be used.
+Only the :obj:`~Orange.classification.tree.TreeLearner.measure` for feature
+scoring and the :obj:`~Orange.classification.tree.TreeLearner.node_learner`
+components have to be chosen so that they work on multi-target data domains.
+
+This module provides one such measure (:class:`MultitargetVariance`) that
+can be used and a helper class :class:`MultiTreeLearner` which extends
+:class:`~Orange.classification.tree.TreeLearner` and is the same in all
+aspects except for different (multi-target) defaults for
+:obj:`~Orange.classification.tree.TreeLearner.measure` and
+:obj:`~Orange.classification.tree.TreeLearner.node_learner`.
+
+Examples
+========
+
+The following example demonstrates how to build a prediction model with
+MultitargetTreeLearner and use it to predict (multiple) class values for
+a given instance (:download:`multitarget.py <code/multitarget.py>`):
+
+.. literalinclude:: code/multitarget.py
+    :lines: 1-4, 10-12
+
+
+.. index:: Multi-target Variance 
+.. autoclass:: Orange.multitarget.tree.MultitargetVariance
+    :members:
+    :show-inheritance:
+
+.. index:: Multi-target Tree Learner
+.. autoclass:: Orange.multitarget.tree.MultiTreeLearner
+    :members:
+    :show-inheritance:
+
+.. index:: Multi-target Tree Classifier
+.. autoclass:: Orange.multitarget.tree.MultiTree
+    :members:
+    :show-inheritance:
+
+"""
+
+from operator import itemgetter
+
+import Orange
+import numpy as np
+
+
+def weighted_variance(X, weights=None):
+    """Computes the variance using a weighted distance to the centroid."""
+    if not weights:
+        weights = [1] * len(X[0])
+    X = X * np.array(weights)
+    return np.sum(np.sum((X - np.mean(X, 0))**2, 1))
+
+class MultitargetVariance(Orange.feature.scoring.Score):
+    """
+    A multi-target score that ranks features based on the average class
+    variance of the subsets.
+
+    To compute it, a prototype has to be defined for each subset. Here, it
+    is just the mean vector of class variables. Then the sum of squared
+    distances to the prototypes is computed in each subset. The final score
+    is obtained as the average of subset variances (weighted, to account for
+    subset sizes).
+    
+    Weights can be passed to the constructor to normalize classes with values
+    of different magnitudes or to increase the importance of some classes. In
+    this case, class values are first scaled according to the given weights.
+    """
+
+    def __init__(self, weights=None):
+        """
+        :param weights: Weights of the class variables used when computing
+                        distances. If None, all weights are set to 1.
+        :type weigts: list
+        """
+
+        # Types of classes allowed
+        self.handles_discrete = True
+        ## TODO: for discrete classes with >2 values entropy should be used
+        ## instead of variance
+        self.handles_continuous = True
+        # Can handle continuous features
+        self.computes_thresholds = True
+        # Needs instances
+        self.needs = Orange.feature.scoring.Score.Generator
+
+        self.weights = weights
+
+
+    def threshold_function(self, feature, data, cont_distrib=None, weights=0):
+        """
+        Evaluates possible splits of a continuous feature into a binary one
+        and scores them.
+        
+        :param feature: Continuous feature to be split.
+        :type feature: :class:`Orange.feature.Descriptor`
+
+        :param data: The data set to be split using the given continuous
+                     feature.
+        :type data: :class:`Orange.data.Table`
+
+        :return: :obj:`list` of :obj:`tuples <tuple>`
+                 [(threshold, score, None),]
+        """
+
+        f = data.domain[feature]
+        values = sorted(set(ins[f].value for ins in data))
+        ts = []
+
+        if values[-1]=='?':
+            values = values[:-1]
+
+        ts = [(v1 + v2) / 2. for v1, v2 in zip(values, values[1:])]
+        if len(ts) > 40:
+            ts = ts[::len(ts)/20]
+        scores = []
+        for t in ts:
+            bf = Orange.feature.discretization.IntervalDiscretizer(
+                points=[t]).construct_variable(f)
+            dom2 = Orange.data.Domain([bf], class_vars=data.domain.class_vars)
+            data2 = Orange.data.Table(dom2, data)
+            scores.append((t, self.__call__(bf, data2)))
+        return scores
+
+    def best_threshold(self, feature, data):
+        """
+        Computes the best threshold for a split of a continuous feature.
+
+        :param feature: Continuous feature to be split.
+        :type feature: :class:`Orange.feature.Descriptor`
+
+        :param data: The data set to be split using the given continuous
+                     feature.
+        :type data: :class:`Orange.data.Table`
+
+        :return: :obj:`tuple` (threshold, score, None)
+        """
+
+        scores = self.threshold_function(feature, data)
+        threshold, score = max(scores, key=itemgetter(1))
+        return (threshold, score, None)
+
+    def __call__(self, feature, data, apriori_class_distribution=None,
+                 weights=0):
+        """
+        :param feature: The feature to be scored.
+        :type feature: :class:`Orange.feature.Descriptor`
+
+        :param data: The data set on which to score the feature.
+        :type data: :class:`Orange.data.Table`
+
+        :return: :obj:`float`
+        """
+
+        split = dict((ins[feature].value, []) for ins in data)
+        for ins in data:
+            split[ins[feature].value].append(ins.get_classes())
+        score = -sum(weighted_variance(x, self.weights) * len(x)
+                     for x in split.values())
+        return score
+
+
+class MultiTreeLearner(Orange.classification.tree.TreeLearner):
+    """
+    MultiTreeLearner is a multi-target version of a tree learner. It is the
+    same as :class:`~Orange.classification.tree.TreeLearner`, except for the
+    default values of two parameters:
+    
+    .. attribute:: measure
+        
+        A multi-target score is used by default: :class:`MultitargetVariance`.
+
+    .. attribute:: node_learner
+        
+        Standard trees use
+        :class:`~Orange.classification.majority.MajorityLearner`
+        to construct prediction models in the leaves of the tree.
+        MultiTreeLearner uses the multi-target equivalent which can be 
+        obtained simply by wrapping the majority learner:
+
+        :class:`Orange.multitarget.MultitargetLearner`
+        (:class:`Orange.classification.majority.MajorityLearner()`).
+
+    """
+
+    def __init__(self, **kwargs):
+        """
+        The constructor passes all arguments to
+        :class:`~Orange.classification.tree.TreeLearner`'s constructor
+        :obj:`Orange.classification.tree.TreeLearner.__init__`.
+        """
+        
+        if 'measure' not in kwargs:
+            kwargs['measure'] = MultitargetVariance()
+        if 'node_learner' not in kwargs:
+            kwargs['node_learner'] = Orange.multitarget.MultitargetLearner(
+                Orange.classification.majority.MajorityLearner())
+        Orange.classification.tree.TreeLearner.__init__(self, **kwargs)
+
+    def __call__(self, data, weight=0):
+        """
+        :param data: Data instances to learn from.
+        :type data: :class:`Orange.data.Table`
+
+        :param weight: Id of meta attribute with weights of instances.
+        :type weight: :obj:`int`
+        """
+        
+        # Use the class, if data does not have class_vars
+        if not data.domain.class_vars and data.domain.class_var:
+            dom = Orange.data.Domain(data.domain.features,
+                data.domain.class_var, class_vars=[data.domain.class_var])
+            data = Orange.data.Table(dom, data)
+
+        # Check for missing class values in data
+        for ins in data:
+            for cval in ins.get_classes():
+                if cval.is_special():
+                    raise ValueError('Data has missing class values.')
+
+        # TreeLearner does not work on class-less domains,
+        # so we set the class if necessary
+        if not data.domain.class_var and data.domain.class_vars:
+            dom = Orange.data.Domain(data.domain.features,
+                data.domain.class_vars[0], class_vars=data.domain.class_vars)
+            data = Orange.data.Table(dom, data)
+
+        tree = Orange.classification.tree.TreeLearner.__call__(
+            self, data, weight)
+        return MultiTree(base_classifier=tree)
+
+class MultiTree(Orange.classification.tree.TreeClassifier):
+    """
+    MultiTree classifier is almost the same as the base class it extends
+    (:class:`~Orange.classification.tree.TreeClassifier`). Only the
+    :obj:`__call__` method is modified so it works with multi-target data.
+    """
+
+    def __call__(self, instance, return_type=Orange.core.GetValue):
+        """
+        :param instance: Instance to be classified.
+        :type instance: :class:`Orange.data.Instance`
+
+        :param return_type: One of
+            :class:`Orange.classification.Classifier.GetValue`,
+            :class:`Orange.classification.Classifier.GetProbabilities` or
+            :class:`Orange.classification.Classifier.GetBoth`
+        """
+
+        node = self.descender(self.tree, instance)[0]
+        return node.node_classifier(instance, return_type)
+
+
+if __name__ == '__main__':
+    data = Orange.data.Table('multitarget-synthetic')
+    print 'Actual classes:\n', data[0].get_classes()
+    
+    majority = Orange.classification.majority.MajorityLearner()
+    mt_majority = Orange.multitarget.MultitargetLearner(majority)
+    c_mtm = mt_majority(data)
+    print 'Majority predictions:\n', c_mtm(data[0])
+
+    mt_tree = MultiTreeLearner(max_depth=3)
+    c_mtt = mt_tree(data)
+    print 'Multi-target Tree predictions:\n', c_mtt(data[0])
 
 ENTRY_POINTS = {
     'orange.addons': (
-        'multitarget2 = _multitarget',
+        'multitarget_addon = _multitarget',
     ),
     #'orange.widgets': (
     #    'Evaluate = _reliability.widgets',