Commits

Anonymous committed 9cb9fc0

add background traffic generator, make use of sadit code, work for existing cases.

Comments (0)

Files changed (14)

core/configure/Network.py

         Dot.__init__(self, 'SimConf', graph_type='graph')
         self.node_list = []
         self.NODE_NUM = 0
-        self.IPSrcSet, self.AnoSet, _ = GetIPAdress()
+        # self.IPSrcSet, self.AnoSet, _ = GetIPAdress()
         self.mv = None
         # self.Node = node_init_handle
 

core/ns3/Experiment/Experiment.py

         botmaster_desc = scen.botmaster_desc
         server_desc = scen.server_desc
         client_desc = scen.client_desc
-        if srv_addr:
+        if srv_addr is not None:
             botmaster_desc['srv_addr'] = srv_addr
             server_desc['srv_addr'] = srv_addr
             client_desc['srv_addr'] = srv_addr
         server_addr_set = self._get_server_addr()
         server_set = self._get_server_nodes()
         for i in xrange(self.node_num):
-            if i in self.server_id_set or self.get_node(i).NODE_TYPE.startswith('real'):
+            # if i in self.server_id_set or self.get_node(i).NODE_TYPE.startswith('real'):
+                # continue
+            if self.get_node(i).NODE_TYPE.startswith('real'):
                 continue
             self.get_node(i).server_addr_set = server_addr_set
             self.get_node(i).server_set = server_set

core/ns3/NS3Config.py

 
 class NS3Config(object):
     """config NS3 according to fs dot file"""
-    TOPOLOGY_FILE = settings.ROOT + '/share/topology.inet'
+    TOPOLOGY_FILE = settings.ROOT + '/share/bg_topology.inet'
     TOPOLOGY_TYPE = 'Inet'
-    NET_SETTINGS_FILE = settings.ROOT + '/share/net_settings_tmp.py'
+    NET_SETTINGS_FILE = settings.ROOT + '/share/bg_net_settings_tmp.py'
     def __init__(self, dot_file, trace_config):
         self.dot_file = dot_file
         self.dot_config = DotConfig(self.dot_file, trace_config)
                 for i in xrange(num):
                     self.add_onoff_app(start_time, end_time, *self.transform_para(**gen))
 
+def TopologyNetBT(dot_file, trace_config):
+    """Create a TopologyNet with Background traffic alread configured
+        - **dotfile** is the path of dot configuration file relative to ROOT directory
+        - **trace_config** is a dictionary contains the trace and bot server client information
+    """
+    # ns3_config = NS3Config(settings.ROOT + '/' + dot_file, trace_config)
+    ns3_config = NS3Config(dot_file, trace_config)
+    ns3_config.setup()
+    ns3_config.config_onoff_app()
+    return ns3_config.net
+
 def run_ns3(dot_file, trace_config, sim_time, visual):
+    """run ns3 simulation based on configuration in **dot_file** and **trace_flag**"""
     if visual:
         ns3.GlobalValue.Bind("SimulatorImplementationType",
                 ns3.StringValue("ns3::VisualSimulatorImpl"))

core/ns3/Node/ImalseNetnsSimNode.py

         """Will set Connect callback function. If succeeded, self.recv will be called. otherwise
         the sock will be closed"""
         server_addr = self._search_server_addr(addr_port[0])
+        print 'addr_port, ', addr_port
         print 'server_addr, ', server_addr
         print 'serval local, ', self.server_addr_set[0].GetLocal()
         # import pdb;pdb.set_trace()

core/ns3/Topology.py

             if i == 0:
                 totnode, totlink = [int(s.strip()) for s in line.rsplit()]
                 continue
-            if i <= totnode: # ignore the positive information
+            if i <= totnode: # ignore the position information
                 continue
 
             _from, _to, _lineBuffer = [s.strip() for s in line.rsplit()]
             link = Link(_fromNode, _from, _toNode, _to)
             self.AddLink(link)
 
+        fid.close()
+
         print( 'finish scanning topology, there are [%i] nodes'%(nodes.GetN()) )
 
         # can only deal with case that node name is string that can be changed int, like '1', '2' ...
         listRH = ns.internet.Ipv4ListRoutingHelper()
         listRH.Add(staticRouting, 0)
         # listRH.Add(nixRouting, 10)
-        listRH.Add(olsr, 10)
+        listRH.Add(olsr, 5)
 
         stack.SetRoutingHelper(listRH)
         stack.Install(nodes)

experiments/ManualTopoBTExperiment.py

+"""
+Manual Topology Experiment with Background Traffic
+
+"""
+from __future__ import print_function
+import settings
+from core.ns3.NS3Config import TopologyNetBT
+from experiments import experiment_factory
+from core.configure import gen_anomaly_dot
+
+ManualTopoExperiment = experiment_factory('ManualTopoExperiment', BaseClass)
+from util import Namespace
+
+zeros = lambda s:[[0 for i in xrange(s[1])] for j in xrange(s[0])]
+def get_inet_adj_mat(fname):
+    fid = open(fname, 'r')
+    i = -1
+    while True:
+        i += 1
+        line = fid.readline()
+        if not line: break
+        if i == 0:
+            totnode, totlink = [int(s.strip()) for s in line.rsplit()]
+            adj_mat = zeros([totnode, totnode])
+            continue
+        if i <= totnode: # ignore the position information
+            continue
+
+        _from, _to, _lineBuffer = [s.strip() for s in line.rsplit()]
+        adj_mat[int(_from)][int(_to)] = 1
+    fid.close()
+
+    return adj_mat
+
+class ManualTopoBTExperiment(ManualTopoExperiment):
+    """This is a extension of manual topology experiment which add background traffic
+    to the network. """
+    DOT_FILE = settings.ROOT + '/net_config/ManualTopoBTTopology.dot'
+    def initparser(self, parser):
+        ManualTopoExperiment.initparser(self, parser)
+
+        parser.set_defaults(back_traf="net_config/back_traf.py",
+                )
+        parser.add_option('--back_traf', dest="back_traf",
+                help='parameters for back ground traffic',
+                )
+
+    def load_back_traf(self, **kwargs):
+        """load parameters for generating the background traffic. **kwargs** contains some
+        additional parameters"""
+        s = kwargs
+        execfile(settings.ROOT + '/' + self.options.back_traf, s)
+        return Namespace(s)
+
+    def load_exper_settings(self, ns):
+        from util import CIDR_to_subnet_mask
+        # botnet related configuration
+        self.server_id_set = ns.server_id_set
+        self.botmaster_id_set = ns.botmaster_id_set
+        self.client_id_set = ns.client_id_set
+        # print('ns.server_addr', ns.server_addr)
+
+        # parse the server address
+        if len(ns.server_addr) == 0:
+            self.SERVER_ADDR = ''
+            self.NETWORK_BASE = ''
+            self.IP_MASK = ''
+        else:
+            self.SERVER_ADDR, self.NETWORK_BASE, self.IP_MASK = CIDR_to_subnet_mask(ns.server_addr[0]);
+
+    # @staticmethod
+    def gen_back_traf_dot(self, net_settings):
+        """generate background traffic dot file
+        """
+        # TODO generate netDesc, normalDesc
+        def get_trace_config(net_settings):
+            """transform net_setting to trace config"""
+            keys = ['pcap_links', 'pcap_nodes',
+                    'server_id_set', 'botmaster_id_set', 'client_id_set',
+                    'server_addr']
+            ns = dict.copy(net_settings)
+            return dict(elem for elem in ns.items() if elem[0] in keys)
+
+        # get back_traf parameter
+        topo = get_inet_adj_mat(settings.ROOT + '/' + self.options.topology_file)
+        back_traf = self.load_back_traf(
+                topo = topo,
+                srv_node_list = net_settings.server_id_set,
+                sim_t = self.options.simtime,
+                )
+
+        # call the SADIT/Configure module to generate the dot file specifying the background
+        # traffic pattern.
+        gen_anomaly_dot(back_traf.ANO_LIST, back_traf.NET_DESC, back_traf.NORM_DESC, self.DOT_FILE)
+        return self.DOT_FILE, get_trace_config(net_settings)
+
+    def setup(self):
+        BaseClass.setup(self)
+        net_settings = self.load_net_settings()
+        self.load_exper_settings(net_settings)
+
+        # back_traf = self.load_back_traf() # get back_traf parameter
+        dot_file, trace_config = self.gen_back_traf_dot(net_settings)
+
+        self.net = TopologyNetBT(dot_file, trace_config)
+
+        self.net.set_trace()
+        self._install_cmds(srv_addr = self.SERVER_ADDR)
+        self.print_srv_addr()
+        self._set_server_info()
+        self.start_nodes()
+

net_config/ManualTopoBTTopology.inet

+graph SimConf {
+n0 [mod_num=0, traffic="", ipdests="10.0.7.4/24 10.0.8.4/24 10.0.9.4/24 10.0.10.4/24 10.0.11.4/24 10.0.12.4/24", autoack="False"];
+n1 [s1_1="harpoon flowstart=exponential(1.000000) ipsrc=10.0.13.4/24 flowsize=normal(4000.000000,100.000000) lossrate=randomchoice(0.001) ipdst=10.0.8.4/24 dport=randomunifint(1025,65535) sport=randomchoice(22,80,443)", mod_num=0, ipdests="10.0.7.5/24 10.0.13.4/24", m1_1="modulator profile=((8000,),(1,)) start=0 generator=s1_1", traffic="m1_1", autoack="False"];
+n2 [mod_num=0, ipdests="10.0.8.5/24 10.0.14.4/24", s2_1="harpoon flowstart=exponential(1.000000) ipsrc=10.0.14.4/24 flowsize=normal(4000.000000,100.000000) lossrate=randomchoice(0.001) ipdst=10.0.12.4/24 dport=randomunifint(1025,65535) sport=randomchoice(22,80,443)", traffic="m2_1", m2_1="modulator profile=((8000,),(1,)) start=0 generator=s2_1", autoack="False"];
+n3 [mod_num=0, ipdests="10.0.9.5/24 10.0.15.4/24", traffic="m3_1", s3_1="harpoon flowstart=exponential(1.000000) ipsrc=10.0.9.5/24 flowsize=normal(4000.000000,100.000000) lossrate=randomchoice(0.001) ipdst=10.0.8.4/24 dport=randomunifint(1025,65535) sport=randomchoice(22,80,443)", m3_1="modulator profile=((8000,),(1,)) start=0 generator=s3_1", autoack="False"];
+n4 [mod_num=0, m4_1="modulator profile=((8000,),(1,)) start=0 generator=s4_1", ipdests="10.0.15.5/24", traffic="m4_1", s4_1="harpoon flowstart=exponential(1.000000) ipsrc=10.0.15.5/24 flowsize=normal(4000.000000,100.000000) lossrate=randomchoice(0.001) ipdst=10.0.7.4/24 dport=randomunifint(1025,65535) sport=randomchoice(22,80,443)", autoack="False"];
+n5 [mod_num=0, s5_1="harpoon flowstart=exponential(1.000000) ipsrc=10.0.10.5/24 flowsize=normal(4000.000000,100.000000) lossrate=randomchoice(0.001) ipdst=10.0.11.4/24 dport=randomunifint(1025,65535) sport=randomchoice(22,80,443)", ipdests="10.0.10.5/24", m5_1="modulator profile=((8000,),(1,)) start=0 generator=s5_1", traffic="m5_1", autoack="False"];
+n6 [mod_num=0, s6_1="harpoon flowstart=exponential(1.000000) ipsrc=10.0.11.5/24 flowsize=normal(4000.000000,100.000000) lossrate=randomchoice(0.001) ipdst=10.0.10.4/24 dport=randomunifint(1025,65535) sport=randomchoice(22,80,443)", ipdests="10.0.11.5/24", m6_1="modulator profile=((8000,),(1,)) start=0 generator=s6_1", traffic="m6_1", autoack="False"];
+n7 [mod_num=0, s7_1="harpoon flowstart=exponential(1.000000) ipsrc=10.0.14.5/24 flowsize=normal(4000.000000,100.000000) lossrate=randomchoice(0.001) ipdst=10.0.11.4/24 dport=randomunifint(1025,65535) sport=randomchoice(22,80,443)", ipdests="10.0.14.5/24", traffic="m7_1", m7_1="modulator profile=((8000,),(1,)) start=0 generator=s7_1", autoack="False"];
+n8 [s8_1="harpoon flowstart=exponential(1.000000) ipsrc=10.0.13.5/24 flowsize=normal(4000.000000,100.000000) lossrate=randomchoice(0.001) ipdst=10.0.8.4/24 dport=randomunifint(1025,65535) sport=randomchoice(22,80,443)", mod_num=0, ipdests="10.0.13.5/24", m8_1="modulator profile=((8000,),(1,)) start=0 generator=s8_1", traffic="m8_1", autoack="False"];
+n9 [mod_num=0, s9_1="harpoon flowstart=exponential(1.000000) ipsrc=10.0.12.5/24 flowsize=normal(4000.000000,100.000000) lossrate=randomchoice(0.001) ipdst=10.0.10.4/24 dport=randomunifint(1025,65535) sport=randomchoice(22,80,443)", ipdests="10.0.12.5/24", traffic="m9_1", m9_1="modulator profile=((8000,),(1,)) start=0 generator=s9_1", autoack="False"];
+n0 -- n1  [delay=0.01, capacity=10000000, weight=10];
+n0 -- n2  [delay=0.01, capacity=10000000, weight=10];
+n0 -- n3  [delay=0.01, capacity=10000000, weight=10];
+n0 -- n5  [delay=0.01, capacity=10000000, weight=10];
+n0 -- n6  [delay=0.01, capacity=10000000, weight=10];
+n0 -- n9  [delay=0.01, capacity=10000000, weight=10];
+n1 -- n8  [delay=0.01, capacity=10000000, weight=10];
+n2 -- n7  [delay=0.01, capacity=10000000, weight=10];
+n3 -- n4  [delay=0.01, capacity=10000000, weight=10];
+}

net_config/back_traf.py

+""" This file will accept
+    **topo** and **srv_node_list** as parameters
+"""
+#################################
+##   topology ##
+#################################
+link_attr = {'weight':'10', 'capacity':'10000000', 'delay':'0.01'} # link Attribute
+
+
+#################################
+##   Parameter For Normal Case ##
+#################################
+sim_t = 8000
+start = 0
+DEFAULT_PROFILE = ((sim_t,),(1,))
+
+
+# gen_desc = {'TYPE':'harpoon', 'flow_size_mean':'4e5', 'flow_size_var':'100', 'flow_arrival_rate':'0.5'}
+gen_desc = {'TYPE':'harpoon', 'flow_size_mean':'4e3', 'flow_size_var':'100', 'flow_arrival_rate':'1'}
+NORM_DESC = dict(
+        TYPE = 'NORMAl',
+        start = '0',
+        node_para = {'states':[gen_desc]},
+        profile = DEFAULT_PROFILE,
+        )
+
+ANO_LIST = []
+
+NET_DESC = dict(
+        topo=topo,
+        # size=topo.shape[0],
+        size=len(topo),
+        srv_list=srv_node_list,
+        link_attr=link_attr,
+        node_type='NNode',
+        node_para={},
+        )

net_config/net_settings.py

 
 
 # botnet related configuration
-server_id_set = [0]
 botmaster_id_set = [1]
 client_id_set = [2, 3, 4]
-server_addr = ["10.0.1.1/24"]
+# server_addr = ["10.0.12.4/24"]
+# you just need to specify either server_id_set or server_addr
+server_id_set = [0]
+server_addr = []

share/bg_net_settings_tmp.py

+ipv4_net_addr_base = '10.7.0.1/24'
+link_attr_default = ['2ms','5Mbps']
+link_attr = {   (0, 1): ['0.01s', '10000000bps'],
+    (0, 2): ['0.01s', '10000000bps'],
+    (0, 3): ['0.01s', '10000000bps'],
+    (0, 5): ['0.01s', '10000000bps'],
+    (0, 6): ['0.01s', '10000000bps'],
+    (2, 7): ['0.01s', '10000000bps'],
+    (3, 4): ['0.01s', '10000000bps'],
+    (8, 1): ['0.01s', '10000000bps'],
+    (9, 0): ['0.01s', '10000000bps']}
+link_to_ip_map = {   (0, 1): ('10.0.7.4/24', '10.0.7.5/24'),
+    (0, 2): ('10.0.8.4/24', '10.0.8.5/24'),
+    (0, 3): ('10.0.9.4/24', '10.0.9.5/24'),
+    (0, 5): ('10.0.10.4/24', '10.0.10.5/24'),
+    (0, 6): ('10.0.11.4/24', '10.0.11.5/24'),
+    (2, 7): ('10.0.14.4/24', '10.0.14.5/24'),
+    (3, 4): ('10.0.15.4/24', '10.0.15.5/24'),
+    (8, 1): ('10.0.13.5/24', '10.0.13.4/24'),
+    (9, 0): ('10.0.12.5/24', '10.0.12.4/24')}
+
+pcap_nodes = [0]
+pcap_links = [(0, 1)]
+botmaster_id_set = [1]
+client_id_set = [2, 3, 4]
+server_id_set = [0]
+server_addr = []

share/bg_topology.inet

+10	9
+8	0.000000	0.000000
+9	0.000000	0.000000
+0	0.000000	0.000000
+1	0.000000	0.000000
+2	0.000000	0.000000
+3	0.000000	0.000000
+4	0.000000	0.000000
+5	0.000000	0.000000
+6	0.000000	0.000000
+7	0.000000	0.000000
+8	1	1
+9	0	1
+0	1	1
+0	2	1
+0	3	1
+0	5	1
+0	6	1
+2	7	1
+3	4	1
 from util import *
 from Namespace import Namespace
 from net import *
+from joint_pdf import *
+from sadit_util import *
 

util/joint_pdf.py

+#!/usr/bin/env python
+"""
+the goal of this file is to provide a lib of joint probability transformation.
+During the transformation, the marginal distribution is unchanged.
+now only consider the 2-d case.
+"""
+def joint_prob_trans(A, d):
+    """
+    - A is the reference joint distribution,
+    - d is the degree of the change.
+    """
+    # P_star = max_entropy(A)
+    # return d * P_star + (1-d) * A
+
+try:
+    from numpy import array, ones, dot, vstack
+    from numpy.ma import log
+    import numpy as np
+except ImportError:
+    print 'numpy cannot be imported'
+
+def H(x, y):
+    """Cross Entropy Function"""
+    x = x.reshape(-1)
+    y = y.reshape(-1)
+    return dot(x, log(x/y))
+
+def eq_cons(x, A, epsilon):
+    t, _ = A.shape
+    v1 = ones([t, 1])
+    P = x.reshape(A.shape)
+    # import pdb;pdb.set_trace()
+    r1 = dot(P, v1) - dot(A, v1)
+    r2 = dot(P.T, v1) - dot(A.T, v1)
+    r3 = H(P, A) - epsilon
+    return vstack([r1, r2, r3.reshape(-1, 1)]).reshape(-1,)
+
+try:
+   from scipy.optimize import fmin_slsqp
+except:
+   print 'no scipy'
+
+def get_diff_jpdf_with_ini(A, P0, epsilon):
+    """Get joint distribution with same marginal distribution with A
+    and cross entropy with A is epsilon."""
+    t, _ = A.shape
+    obj_func = lambda x: dot(x, log(x))
+    f_eqcons = lambda x: eq_cons(x, A, epsilon)
+    out = fmin_slsqp(func = obj_func,
+            x0 = P0.reshape(-1,),
+            f_eqcons = f_eqcons,
+            bounds = [[0, 1]] *t*t,
+            )
+    return  out.reshape(A.shape)
+
+def get_diff_jpdf(A, epsilon):
+    return get_diff_jpdf_with_ini(A, A, epsilon)
+
+
+def test_p2():
+    print '-' * 40
+    A = array([
+        [0.1, 0.1, 0.1],
+        [0.1, 0.1, 0.1],
+        [0.1, 0.1, 0.2],
+        ])
+    out = get_diff_jpdf_with_ini(A, A, 0.7)
+    print 'out, ', out
+    print 'sum, ', np.sum(out)
+    print '-' * 40
+
+if __name__ == "__main__":
+    test_p2()
+
+

util/sadit_util.py

+from random import randint
+
+### -- [2012-03-04 12:12:42] add binary_search
+### -- [2012-03-26 14:01:02] add docstring for each function.
+
+def IN(*val_list):
+    """Generate a string command that import object variables
+    to locals() in the class methods"""
+    return ";".join(['%s=self.%s'%(v, v) for v in val_list])
+
+def OUT(*val_list):
+    """Generate a string command that export object variables
+    to locals() in the class methods"""
+    return ";".join(['self.%s=%s'%(v, v) for v in val_list])
+
+def binary_search(a, x, lo=0, hi=None):
+    """
+    Find the index of largest value in a that is smaller than x.
+    a is sorted Binary Search
+    """
+    # import pdb;pdb.set_trace()
+    if hi is None: hi = len(a)
+    while lo < hi:
+        mid = (lo + hi) / 2
+        midval = a[mid]
+        if midval < x:
+            lo = mid + 1
+        elif midval > x:
+            hi = mid
+        else:
+            return mid
+    return hi-1
+
+Find = binary_search
+
+import types
+def Load(var):
+    '''Load is useful when the some elements in var is specified as random value.
+    for example, if the var is ['rand(1)', 1], var[0] will be different random
+    value at each time.'''
+    t = type(var)
+    if t == types.TupleType or t == types.ListType:
+        return [Load(x) for x in var]
+    elif t == types.DictType:
+        res = dict()
+        for k, v in var.iteritems():
+            # If cannot properly loaded, use origianl value
+            try:
+                res[k] = Load(v)
+            except Exception as e:
+                res[k] = v
+        return res
+    elif t == types.StringType:
+        return eval(var)
+    elif t == types.FloatType or t == types.IntType:
+        return var
+    else:
+        raise TypeError('unknown type of var')
+
+def Dump2Txt(var, fname, op):
+    """Dump2Txt will dump the variable to text file for use of other programs like Matlab.
+
+    - *fname* : is the name for output file
+    - *op* : is a option flag, ::
+
+        if op[0:2] == '1d':
+            m = len(var)
+            for i in xrange(m): fid.write("%f "%(var[i]))
+            fid.write('\\n')
+        elif op[0:2] == '2d':
+            if op[2:] == 'np': m, n = var.shape
+            elif op[2:] == 'list':
+                m = len(val)
+                m = len(val[0])
+            else:
+                raise ValueError('unknown op')
+
+            for i in xrange(m):
+                for j in xrange(n):
+                    fid.write("%s "%(var[i,j]))
+                fid.write("\\n")
+
+
+    """
+    fid = open(fname, 'w')
+    if op[0:2] == '1d':
+        m = len(var)
+        for i in xrange(m): fid.write("%f "%(var[i]))
+        fid.write('\n')
+    elif op[0:2] == '2d':
+        if op[2:] == 'np': m, n = var.shape
+        elif op[2:] == 'list':
+            m = len(val)
+            m = len(val[0])
+        else:
+            raise ValueError('unknown op')
+
+        for i in xrange(m):
+            for j in xrange(n):
+                fid.write("%s "%(var[i,j]))
+            fid.write("\n")
+    else:
+        raise ValueError('unknow op')
+
+    fid.close()
+
+
+import imp
+def CreateSettings(templateFilePath, settingsFilePath, **kwargs):
+    '''settings.py is a file contains all the global parameters.
+    Sometimes we need to do the sensitivity analysis and need to change the global
+    parameter on the fly. CreateSetting will faciliate the process through generating
+    settings.py based on a template file.
+    file.
+    '''
+    settings_template = imp.load_source('settings_template', templateFilePath)
+    namespace = settings_template.__dict__
+    for k, v in kwargs.iteritems():
+        namespace[k] = v
+    PrintVar(namespace, settingsFilePath)
+
+import types
+imports = 'types', 'sys', 'PrintVar', 'os', 'settings'
+
+try:
+    import numpy as np
+except ImportError:
+    print 'no numpy'
+
+def PrintVar(namespace, outputFile = ''):
+    '''Print all variances in the namespace into .py file'''
+    fid = -1
+    if outputFile != '':
+        fid = open(outputFile, 'w')
+    import inspect
+    for k, v in namespace.iteritems():
+        if k.startswith("__")==0 and k not in imports:
+            # print 'type(v), ', type(v)
+            if type(v) == types.StringType:
+                expr ='%s = \'%s\'\n' %(k, str(v))
+            elif type(v) == types.FunctionType:
+                expr = inspect.getsource(v) + '\n'
+                # removing the leading blankspace
+                leadingSpace = expr.rfind('def')
+                if leadingSpace != 0 and leadingSpace != -1:
+                    srcLine = inspect.getsourcelines(v)
+                    expr = ''
+                    for line in srcLine[0]:
+                        expr = expr + line[leadingSpace:]
+                if leadingSpace != -1:
+                    GetFuncName = lambda s: s[s.find('def')+4:s.find('(')]
+                    funcName = GetFuncName(expr)
+                    if funcName != k: expr += '\n%s = %s\n' %(k, funcName)
+
+            elif type(v) == types.BuiltinFunctionType:
+                module =inspect.getmodule(v)
+                expr = 'from %s import %s\n' %(module.__name__,  v.__name__)
+            elif type(v) == types.ModuleType:
+                expr = 'import %s as %s\n' %(v.__name__, k)
+            elif type(v) == np.ndarray:
+                expr = k + ' = ' + str(v.tolist()) + '\n'
+            else:
+                expr = '%s = %s\n' %(k, str(v))
+            if fid == -1:
+                print expr,
+                continue
+            fid.write( expr )
+    if fid != -1:
+        fid.close()
+
+
+def PrintModelFree(mfIndi, mbIndi):
+    '''Print the ModelFree Derivative which is not nan value'''
+    # mfIndi = ModelFreeDetectAnoType()
+    # mbIndi = ModelBaseDetectAnoType()
+    for i in xrange(len(mfIndi)):
+        if not np.isnan( mfIndi[i]):
+            print '[%d]\t%f'%(i, mfIndi[i])
+    print '\n'
+
+
+def PrintModelBase(mbIndi):
+    '''print the model based derivative which is not nan value.'''
+    m, n = mbIndi.shape
+    for i in xrange(m):
+        for j in xrange(n):
+            if not np.isnan(mbIndi[i,j]):
+                print '[%d, %d]\t%f' %(i, j, mbIndi[i,j])
+    print '\n'
+
+
+def abstract_method():
+    """ This should be called when an abstract method is called that should have been
+    implemented by a subclass. It should not be called in situations where no implementation
+    (i.e. a 'pass' behavior) is acceptable. """
+    raise NotImplementedError('Method not implemented!')
+
+def FROM_CLS(*val_list):
+    return ";".join(['%s=self.%s'%(v, v) for v in val_list])
+
+def TO_CLS(*val_list):
+    return ";".join(['self.%s=%s'%(v, v) for v in val_list])
+
+
+class DataEndException(Exception):
+    pass
+
+class FetchNoDataException(Exception):
+    pass
+
+
+
+QUAN = 1
+NOT_QUAN = 0
+
+# The Distance Function
+DF = lambda x,y:abs(x[0]-y[0]) * (256**3) + abs(x[1]-y[1]) * (256 **2) + abs(x[2]-y[2]) * 256 + abs(x[3]-y[3])
+
+def zeros(s):
+    if len(s) == 1:
+        return [0] * s[0]
+    elif len(s) == 2:
+        return [[0 for i in xrange(s[1])] for j in xrange(s[0])]
+    else:
+        raise Exception('unknown size in zeros')
+
+
+import inspect
+def get_help_docs(dic):
+    docs = []
+    for k, v in dic.iteritems():
+        doc  = inspect.getdoc(v)
+        comp_doc = "%s %s"%(v.__name__, doc) if doc else v.__name__
+        docs.append("'%s': %s"%(k, comp_doc))
+
+    return docs