Commits

Jing Wang committed 0a9f945

add code to export abnormal flows

  • Participants
  • Parent commits 0393e71

Comments (0)

Files changed (49)

.UNDO/settings.py_undo

Binary file added.

Configure/API.pyc

Binary file added.

Configure/Anomaly (wangjing's conflicted copy 2012-05-14).py

+#!/usr/bin/env python
+
+### -- [2012-03-04 14:56:03] FlowRate and FlowSize anomaly parameter has be changed as
+### -- ratio instead of absolute value
+##-- [2012-04-08 22:31:20] Add GenAnomalyDot
+##-- [2012-04-09 18:31:22] refactoring the whole file
+## -- [2012-04-10 01:14:07] FLOW_RATE can work
+##-- [2012-04-10 17:16:27] add _infect_modulator, make anomaly more general
+
+
+import numpy as np
+import sys
+sys.path.append("..")
+import settings
+from util import Load
+from mod_util import choose_ip_addr
+import cPickle as pickle
+
+from numpy import cumsum, diff
+def get_pos(l, v):
+    """index of largest element in l that is less than v"""
+    for i in xrange(len(l)):
+        if l[i] < v :
+            continue
+        return i - 1
+
+def insert_break_pt(b, dur, num):
+    """b is a break point that will break dur, for example,
+    if b = 35, and dur = (20, 20, 10), num = (1, 2, 1)the result will be
+        (20, 15, 5, 10), the new num will be (1, 2, 2, 1)"""
+    t = [0] + list(cumsum( dur ))
+    nt = copy.deepcopy(t)
+    new_num = list(copy.deepcopy(num))
+    i = get_pos(t, b)
+    if i == -1 or i == len(t) - 1:
+        return dur, num, i+1;
+    else:
+        nt.insert(i+1, b)
+        new_num.insert(i+1, num[i])
+        new_dur = list(diff(nt))
+        return new_dur, new_num, i+1
+
+class BadConfigError(Exception):
+    def __init__(self, value):
+        self.value = value
+    def __str__(self):
+        return repr(self.value)
+
+
+import copy
+class Anomaly:
+    '''basis class for anomaly. Its subclass will provide run() method'''
+    def __init__(self, ano_desc):
+        self.ano_desc = ano_desc
+        self.ano_node = None
+
+    def get_profile_with_ano(self, mod_start, mod_profile, ano_t):
+        """in fs, one modulator can only one behaviour
+        toe simulate the change of behaviour of the modulator,
+        the abnormal haviour will be generator by new modulators."""
+        start, end = ano_t
+        start -= mod_start
+        end -= mod_start
+        d, n, i1 = insert_break_pt(start, mod_profile[0], mod_profile[1])
+        d, n, i2 = insert_break_pt(end, d, n)
+        normal_profile_1 = ( tuple(d[:i1]), tuple(n[:i1]) )
+        abnormal_profile = ( tuple(d[i1:i2]), tuple(n[i1:i2]) )
+        normal_profile_2 = ( tuple(d[i2:]), tuple(n[i2:]) )
+        return normal_profile_1, abnormal_profile, normal_profile_2
+
+    def cut_profile(profile, status):
+        """cut into three pieces"""
+
+    def _infect_modulator(self, ano_t, m_id, mod):
+        ano_node = self.ano_node
+        generator = ano_node.generator
+
+        mod_start = eval(mod['start'])
+        mod_profile = mod['profile']
+        np1, ap, np2 = self.get_profile_with_ano(mod_start, mod_profile, ano_t)
+
+        s_id = mod['generator'] # get id for source generator
+        ano_node.add_modulator(start=str(mod_start), profile=np1, generator = [generator[s_id]])
+
+        start, end = ano_t
+        st = mod_start + float(np.sum(np1[0]))
+        assert(st == start)
+
+        # print self.ano_desc['change']
+        # import pdb;pdb.set_trace()
+        new_generator = generator[s_id].get_new_gen(self.ano_desc['change'])
+        pickle.dump(new_generator.para, open(settings.EXPORT_ABNORMAL_FLOW_PARA_FILE, 'w'))
+        ano_node.add_modulator(start=str(start),
+                profile=ap,
+                generator = [ new_generator ])
+
+        st = mod_start + float(np.sum(np1[0])) + float(np.sum(ap[0]))
+        assert(st == end)
+        ano_node.add_modulator(start=str(end), profile=np2, generator=[ generator[s_id] ])
+
+        # delete original modulator
+        del ano_node.modulator[m_id]
+        del ano_node.generator[s_id]
+
+    def run(self, net):
+        """inject itself into the network"""
+        self.ano_node = net.node_list[self.ano_desc['ano_node_seq']]
+        ano_t = self.ano_desc['T']
+
+        m_back = copy.deepcopy(self.ano_node.modulator)
+        for m_id, mod in m_back.iteritems(): # For each modulator
+            self._infect_modulator(ano_t, m_id, mod)
+
+
+from Edge import NEdge
+from Node import NNode
+from Generator import get_generator
+class AtypicalUserAnomaly(Anomaly):
+    """anomaly of atypical user. an atypical user joins to the network during some time.
+    Atypical user refer those user has large IP distance with users in the network."""
+    ATIP = None # Atypical IP Set. Will Select IP from this set and add a node with atypical ip
+    idx = 0 # A indicator to seperate the IP that has been selected or not
+    NAME = 'AtypicaUser'
+    def __init__(self, ano_desc):
+        """link_to is a list of variables representing the connection to all other nodes
+        * link_to[i] == 1 means there is link from atypical node to node i.
+        * link_to[i] == -1 means there is link from node i to this atypical node.
+        """
+        self.ano_desc = ano_desc
+        Anomaly.__init__(self, ano_desc)
+        if self.ATIP == None:
+            self.ATIP = ano_desc['ATIP']
+        self.net = None
+        self.ano_node = None
+
+    def _change_topology(self):
+        link_to = self.ano_desc['link_to']
+        link_attr = self.ano_desc['link_attr']
+        for i in xrange(len(link_to)):
+            if link_to[i] == 1:
+                edge = NEdge(self.ano_node, self.net.node_list[i], link_attr )
+            elif link_to[i] == -1:
+                edge = NEdge(self.net.node_list[i], self.ano._ode, link_attr )
+            else:
+                raise ValueError('unknown link_to value')
+            self.net.add_edge(edge)
+
+    def _config_traffic(self):
+        nn = len(self.net.node_list)
+        srv_node_list = [self.net.node_list[i] for i in xrange(nn) if i in self.net.net_desc['srv_list'] ]
+        start, end = self.ano_desc['T']
+        for srv_node in srv_node_list:
+            gen_desc = Load(self.ano_desc['gen_desc'])
+            gen_desc['ipsrc'] = choose_ip_addr(self.ano_node.ipdests)
+            gen_desc['ipdst'] = choose_ip_addr(srv_node.ipdests)
+            self.ano_node.add_modulator(start=str(start),
+                    profile='((%d,),(1,))' %(end-start),
+                    generator=get_generator(gen_desc) )
+
+    def _get_ano_node(self):
+        ipdest = [ self.ATIP[self.idx] ]
+        self.idx += 1
+        self.ano_node = NNode(ipdest)
+        self._config_traffic()
+
+    def _export_ip_addr(self):
+        fid = open(settings.ATYPICAL_IP_FILE, 'w')
+        fid.write( ' '.join([str(i) for i in self.ano_node.ipdests]) )
+        fid.close()
+
+    def run(self, net):
+        '''will add a node for atypical user to the network.
+        The IP address for atypical user is selected from. settings.atypical_ip_file'''
+        self.net = net
+        self._get_ano_node()
+        net.add_node(self.ano_node)
+        self._change_topology()
+
+        self._export_ip_addr()
+
+class TargetOneServer(Anomaly):
+    """Only change the behaviour in one server
+    ano_desc should have id **srv_id** of that sever node"""
+    def run(self, net):
+        self.ano_node = net.node_list[self.ano_desc['ano_node_seq']]
+        ano_t = self.ano_desc['T']
+        srv_id = self.ano_desc['srv_id']
+        srv_ip_addr = net.node_list[srv_id].ipdests
+        m_back = copy.deepcopy(self.ano_node.modulator)
+        for m_id, mod in m_back.iteritems(): # For each modulator
+            s_id = mod['generator'] # get id for source generator
+            if self.ano_node.generator[s_id]['ipdst'] not in srv_ip_addr:
+                continue
+            self._infect_modulator(ano_t, m_id, mod)
+
+

Configure/Anomaly.py

 import settings
 from util import Load
 from mod_util import choose_ip_addr
+import cPickle as pickle
 
 from numpy import cumsum, diff
 def get_pos(l, v):
         st = mod_start + float(np.sum(np1[0]))
         assert(st == start)
 
-        # print self.ano_desc['change']
-        # import pdb;pdb.set_trace()
+        self.new_generator = generator[s_id].get_new_gen(self.ano_desc['change'])
         ano_node.add_modulator(start=str(start),
                 profile=ap,
-                generator = [ generator[s_id].get_new_gen(self.ano_desc['change']) ])
+                generator = [ self.new_generator ])
 
         st = mod_start + float(np.sum(np1[0])) + float(np.sum(ap[0]))
         assert(st == end)
         del ano_node.modulator[m_id]
         del ano_node.generator[s_id]
 
+    def _export_ano_flow_para(self):
+        """export para to help to export ano flows"""
+        pickle.dump(self.new_generator.para, open(settings.EXPORT_ABNORMAL_FLOW_PARA_FILE, 'w')) # For export abnormal flows
+
     def run(self, net):
         """inject itself into the network"""
         self.ano_node = net.node_list[self.ano_desc['ano_node_seq']]
         for m_id, mod in m_back.iteritems(): # For each modulator
             self._infect_modulator(ano_t, m_id, mod)
 
+        self._export_ano_flow_para()
+
 
 from Edge import NEdge
 from Node import NNode
         self._config_traffic()
 
     def _export_ip_addr(self):
-        fid = open(settings.ATYPICAL_IP_FILE, 'w')
+        fid = open(settings.EXPORT_ABNORMAL_FLOW_PARA_FILE, 'w')
         fid.write( ' '.join([str(i) for i in self.ano_node.ipdests]) )
         fid.close()
 
+    def _export_ano_flow_para(self):
+        """export para to help to export ano flows"""
+        self._export_ip_addr()
+
     def run(self, net):
         '''will add a node for atypical user to the network.
         The IP address for atypical user is selected from. settings.atypical_ip_file'''
         net.add_node(self.ano_node)
         self._change_topology()
 
-        self._export_ip_addr()
+        self._export_ano_flow_para()
 
 class TargetOneServer(Anomaly):
     """Only change the behaviour in one server
             if self.ano_node.generator[s_id]['ipdst'] not in srv_ip_addr:
                 continue
             self._infect_modulator(ano_t, m_id, mod)
-
-

Configure/Anomaly.pyc

Binary file added.

Configure/Behaviour.pyc

Binary file added.

Configure/Edge.pyc

Binary file added.

Configure/Generator.pyc

Binary file added.

Configure/MVAnomaly.pyc

Binary file added.

Configure/MarkovAnomaly.pyc

Binary file added.

Configure/Modulator.pyc

Binary file added.

Configure/Network.pyc

Binary file added.

Configure/Node.pyc

Binary file added.

Configure/__init__.pyc

Binary file added.

Configure/mod_util.pyc

Binary file added.

Detector/AnomalyDetector.pyc

Binary file added.

Detector/ClusterAlg.pyc

Binary file added.

Detector/DataFile.pyc

Binary file added.

Detector/DataParser.pyc

Binary file added.

Detector/Derivative.pyc

Binary file added.

Detector/DetectorLib.pyc

Binary file added.

Detector/__init__.pyc

Binary file added.

Experiment/.UNDO/Sens.py_undo

Binary file added.

Experiment/Experiment.py

 import numpy as np
 
 class Experiment(object):
+    """
+    base class for all experiments. Three important attributes
+        - net_desc: description of the network
+        - norm_desc: description of the normal case
+        - ano_list: list of anomalies.
+    settings module is used to initialize the experiment, we need have a
+    settings.py file with attribute 1. **NET_DESC** 2. **NORM_DESC** 3. **ANO_LIST**
+    """
     def __init__(self, settings):
         self.settings = settings
     @property

Experiment/Experiment.pyc

Binary file modified.

Experiment/MarkovExperiment.py

         self.net_desc['node_type'] = 'MarkovNode'
         self.net_desc['node_para'] = {
                 'P':Pi2P(self.normal_sta_prob),
-                'interval':100,
+                'interval':10,
                 }
 
     def get_norm_desc(self):
         ano_desc = {'anoType':'markov_anomaly',
                 'ano_node_seq':2,
                 'T':(1200, 1400),
-                'ano_markov_desc':{'P':ano_markov_prob, 'interval':100},
+                'ano_markov_desc':{'P':ano_markov_prob, 'interval':10},
                 'srv_id':0,
                 }
         self.ano_list = [ano_desc]

Experiment/Sens.py

 from Experiment import AttriChangeExper, gen_anomaly_dot
 from matplotlib.pyplot import figure, plot, show, subplot, title, legend
 import cPickle as pickle
+import copy
 
-import copy
 class SensExper(AttriChangeExper):
+    """SensExper is experiment to get the sensitivity result.
+    it will change parameters and run the simulation for several times
+    and plot the both model based and model free entropy
+    ex:
+    .. code-block:: python
+
+        import settings
+        exper = SensExper(settings)
+        exper.run('flow_arrival_rate', [2, 4, 6])
+        exper.plot_entropy()
+
+    """
     def __init__(self, settings):
         AttriChangeExper.__init__(self, settings)
         self.sens_ano = settings.ANO_LIST[0]
         self.shelve_file = '/home/jing/det_obj.out'
 
     def run(self, attr, rg):
+        """attr is the name of attribute that will be changed. possible
+        attrs are :
+            - flow_arrival_rate
+            - flow_size_mean
+            - flow_size_var
+        rg is the list of possible values for *attr*.
+        """
         det_obj_shelf = dict()
-        # self.sens_ano['ano_type'] = 'flow_arrival_rate'
         self.sens_ano['ano_type'] = attr
-        # figure()
-        # for i in [2, 4, 6]:
         for i in rg:
-            # self.sens_ano['change']['flow_arrival_rate'] = i;
             self.sens_ano['change'][attr] = i;
-            # self.ano_list[0]['change']['flow_arrival_rate'] = i;
             self.configure()
             self.simulate()
             det_obj = copy.deepcopy( self.detect() )
             det_obj_shelf[str(i)] = dict(winT=det_obj.record_data['winT'],
                     entropy=det_obj.record_data['entropy'])
-            v = det_obj.record_data['entropy']
-            # plot(zip(*v)[0])
+            # v = det_obj.record_data['entropy']
             self.clear_tmp_file()
+            self.store_abnormal_flow_file(str(i))
 
         f_obj = open(self.shelve_file, 'w')
         pickle.dump(det_obj_shelf, f_obj)
         f_obj.close()
 
+    def store_abnormal_flow_file(self, suffix):
+        import settings, shutil
+        file_name = settings.ROOT + '/Share/abnorma_flow_%s.txt'%(suffix)
+        shutil.copyfile(settings.EXPORT_ABNORMAL_FLOW_FILE, file_name)
+
     def clear_tmp_file(self):
         import os
         os.system('rm %s'%(self.dot_file))
 
     def configure(self):
         gen_anomaly_dot([self.sens_ano], self.net_desc, self.norm_desc, self.dot_file)
-        # gen_anomaly_dot(self.ano_list, self.net_desc, self.norm_desc, self.dot_file)
 
 if __name__ == "__main__":
     import settings

Share/abnorma_flow_2.txt

Empty file added.

Share/abnorma_flow_4.txt

Empty file added.

Share/abnorma_flow_6.txt

Empty file added.

Share/ano_flow_para.txt

+(dp1
+S'ipdst'
+p2
+S'127.126.151.10'
+p3
+sS'ipsrc'
+p4
+S'127.126.44.111'
+p5
+sS'flow_arrival_rate'
+p6
+F3
+sS'flow_size_var'
+p7
+I100
+sS'TYPE'
+p8
+S'harpoon'
+p9
+sS'flow_size_mean'
+p10
+F2400000
+s.