Commits

Anonymous committed 79fe5d3

Initial commit of dad. A cherrypy based eggmonster replacement

Comments (0)

Files changed (12)

dad/__init__.py

Empty file added.

dad/lib/__init__.py

Empty file added.

dad/lib/config.py

+import os
+from collections import defaultdict
+import yaml
+
+class EggmonsterConfigError(Exception): pass
+
+def get_nodes(name, app, options):
+    nodes_s = set()
+    if not options.get('Enabled', True):
+        return nodes_s
+
+    nodes = app.get('Nodes', {})
+
+    for host, count in nodes.iteritems():
+        if not type(count) is int:
+            raise EggmonsterConfigError("Non-integer value given for node count for app %r" % name)
+        for x in xrange(count):
+            nodes_s.add((host, x + 1))
+
+    return nodes_s
+
+def get_pkginfo(pkg, pkgmeta):
+    pkginfo = pkgmeta.get('Package', '')
+    if not pkginfo:
+        return ''
+    pkginfo = '%s %s' % (pkg, pkginfo)
+    return pkginfo
+
+def get_appgroup(pkg, pkgmeta):
+    appgroups = pkgmeta.get('Applications')
+    if not appgroups:
+        raise EggmonsterConfigError("No 'Applications' given for package %r" % pkg)
+    return appgroups
+
+def get_options(cfg):
+    return cfg.get('Options', {})
+
+def get_instgroup(app, appmeta):
+    return appmeta.get('Instances', [])
+
+def get_env(pkgmeta):
+    opts = pkgmeta.get('Environment', {})
+    return opts
+
+def app_nameinfo(pkg, app):
+    name = app.get('Name')
+    if not name:
+        raise EggmonsterConfigError("No 'Name' given for app in package %r" % pkg)
+    entry_point = name
+    return entry_point, '%s.%s' % (pkg, name)
+
+def inst_nameinfo(app, inst):
+    name = inst.get('Name')
+    if not name:
+        raise EggmonsterConfigError("No 'Name' given for app in instance %r" % pkg)
+
+    # Returns entry point without package name, and full application name.
+    return '%s.%s' % (app, name)
+
+class NodeInfo(object):
+    def __init__(self, app_name, n, entry_point, env, options, pkginfo):
+        self.app_name = app_name # full qualified name ("pkg.appname[.instname]")
+        self.n = n # instance number of app_name that is running on the host
+        self.entry_point = entry_point # application to execute ("appname").
+        self.env = env # dictionary containing environment
+        self.options = options # dictionary containing eggmonster running configuration
+        self.pkginfo = pkginfo # string containing which package to use
+
+    @property
+    def key(self):
+        return self.app_name, self.n
+
+    @property
+    def cmp_attributes(self):
+        return self.app_name, self.n, self.entry_point, self.env, self.options, self.pkginfo
+
+    def __cmp__(self, other):
+        if not isinstance(other, NodeInfo):
+            return False
+        return cmp(self.cmp_attributes, other.cmp_attributes)
+
+    def __str__(self):
+        return '<NodeInfo %r>' % (self.cmp_attributes,)
+
+    def __repr__(self):
+        return '<NodeInfo %r>' % (self.cmp_attributes,)
+
+class ClusterConfig(object):
+    def __init__(self, cobject):
+        self.load(cobject)
+
+    def load(self, cobject):
+        self.host_nodes = defaultdict(dict)
+
+        # Keys - application name with package.
+        # Values: tuple of
+        #   - application name without package.
+        #   - environment for application.
+        #   - package version information.
+        #   - options for application. (eggmonster internal)
+        self.apps = {}
+
+        # Keys - name of package.
+        # Values: tuple of:
+        #   - environment for package.
+        #   - package version information.
+        self.packages = {}
+
+        if not cobject:
+            return
+
+        for pkg, pkgmeta in cobject.iteritems():
+            pkginfo = get_pkginfo(pkg, pkgmeta)
+            pkgconf = get_env(pkgmeta)
+            pkgoptions = get_options(pkgmeta)
+            self.packages[pkg] = (pkgconf, pkginfo)
+
+            appgroup = get_appgroup(pkg, pkgmeta)
+            for app in appgroup:
+                app_entry_point, app_name = app_nameinfo(pkg, app)
+
+                app_env = pkgconf.copy()
+                app_env.update(get_env(app))
+
+                app_pkginfo = get_pkginfo(pkg, app) or pkginfo
+
+                app_options = pkgoptions.copy()
+                app_options.update(get_options(app))
+
+                # Process application nodes.
+                nodes = get_nodes(app_name, app, app_options)
+                for host, n in nodes:
+                    ni = NodeInfo(app_name, n, app_entry_point, app_env, app_options, app_pkginfo)
+                    self.host_nodes[host][ni.key] = ni
+                self.apps[app_name] = (app_entry_point, app_env, app_pkginfo, app_options)
+
+                # Process instance nodes.
+                for inst in get_instgroup(app_name, app):
+                    inst_pkginfo = get_pkginfo(pkg, inst) or app_pkginfo
+
+                    inst_env = app_env.copy()
+                    inst_env.update(get_env(inst))
+
+                    inst_options = app_options.copy()
+                    inst_options.update(get_options(inst))
+
+                    inst_name = inst_nameinfo(app_name, inst)
+                    for host, n in get_nodes(inst_name, inst, inst_env):
+                        ni = NodeInfo(inst_name, n, app_entry_point, inst_env, inst_options, inst_pkginfo)
+                        self.host_nodes[host][ni.key] = ni
+
+                    self.apps[inst_name] = (app_entry_point, inst_env, inst_pkginfo, inst_options)
+
+        return
+
+    @classmethod
+    def from_yaml(cls, stream):
+        """
+        Load config from a stream (or string)
+        """
+        return cls(yaml.load(stream))
+
+    @classmethod
+    def from_file(cls, filename):
+        """
+        Load config from a pathname
+        """
+        return cls.from_yaml(open(filename))
+
+# for backward compatibility
+load_config = ClusterConfig.from_file
+load_config_from_yaml = ClusterConfig.from_yaml
+
+
+if __name__ == '__main__':
+    filename = os.path.join(os.path.dirname(__file__),
+        'test', 'test1.yaml')
+    conf = ClusterConfig.from_file(filename)
+    host_nodes = {}
+    for host, host_dict in conf.host_nodes.items():
+        host_nodes[host] = {}
+        for ni_key, ni in host_dict.items():
+            host_nodes[host][ni_key] = ni.__dict__
+
+    print 'HOST NODES:'
+    import pprint
+    pprint.pprint(host_nodes)
+
+    print 'APPS:'
+    pprint.pprint(conf.apps)

dad/web/__init__.py

Empty file added.

dad/web/control/__init__.py

Empty file added.

dad/web/control/main.py

+import cherrypy
+
+from dad.lib.config import load_config
+
+from pprint import pformat
+
+
+class EggmonsterMaster(object):
+    def __init__(self, conf_file):
+        self.conf = load_config(conf_file)
+        cherrypy.engine.publish('dad.reload_conf', self.conf)
+    
+    @cherrypy.expose
+    @cherrypy.tools.json_out()
+    def index(self):
+        return self.conf.apps
+
+    @cherrypy.expose
+    def conf(self):
+        return pformat(self.conf)

dad/web/control/root.py

+import os
+import cherrypy
+import argparse
+
+from .main import EggmonsterMaster
+from dad.web.plugins import manager
+from pprint import pformat
+
+def get_args(*args):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('init_conf', help='The initial configuration file')
+
+    args = parser.parse_args(*args)
+    return args
+
+
+def run(*args):
+    args = get_args(*args)
+
+    here = os.path.dirname(os.path.abspath(os.curdir))
+    confdir = os.path.join(here, 'app_configs')
+    if not os.path.exists(confdir) or not os.path.isdir(confdir):
+        os.mkdir(confdir)
+    
+    cherrypy.config.update({
+        'server.socket_host': 'localhost',
+        'server.socket_port': 5000,
+        'dad.configdir': confdir,
+    })
+    
+    cherrypy.engine.manager = manager.ProcessManagerPlugin(cherrypy.engine)
+    cherrypy.engine.manager.subscribe()
+    
+    cherrypy.engine.config_mgr = manager.ConfigManagerPlugin(cherrypy.engine)
+    cherrypy.engine.config_mgr.subscribe()
+    
+    cherrypy.tree.mount(EggmonsterMaster(args.init_conf), '/')
+    
+    cherrypy.engine.start()
+    
+    cherrypy.engine.block()
+
+
+if __name__ == '__main__':
+    run()

dad/web/plugins/__init__.py

Empty file added.

dad/web/plugins/manager.py

+import cherrypy
+import os
+import sys
+
+import yaml
+
+from cherrypy.process.plugins import SimplePlugin, Monitor
+from contextlib import contextmanager
+from subprocess import Popen, PIPE, STDOUT
+
+from pprint import pprint, pformat
+
+
+class ConfigManagerPlugin(SimplePlugin):
+    def __init__(self, bus):
+        SimplePlugin.__init__(self, bus)
+        self.conf = None
+        self.bus.subscribe('dad.reload_conf', self.reload_conf)
+
+
+    def reload_conf(self, new_conf):
+        self.conf = new_conf
+        for app in self.conf.apps:
+            for host in self.conf.host_nodes:
+                self.bus.publish('dad.start_app', host, self.conf.apps[app])
+
+
+class ProcessManagerPlugin(SimplePlugin):
+    def __init__(self, bus):
+        SimplePlugin.__init__(self, bus)
+        self.conf = None
+        self.processes = {}
+        self.bus.subscribe('dad.start_app', self.start_app)
+
+    def config(self, conf):
+        name, env, pkg_version, options = conf
+        confpath = os.path.join(cherrypy.config['dad.configdir'], '{}.yaml'.format(name))
+        with open(confpath, 'w+') as config:
+            yaml.dump(env, config)
+        return confpath
+
+    def start_app(self, id, appconf):
+        '''
+        We start the app by attaching a new Monitor to the cp
+        engine that listens for its specific set of events.
+        '''
+        conf_path = self.config(appconf)
+        conf = yaml.load(open(conf_path))
+        cmd = conf['script'].format(conf=conf_path)
+        proc = Popen(cmd.split(), stdout=sys.stdout, stderr=STDOUT)
+        self.processes[id] = proc
+
+
+            

example/example.yaml

+example_app: # the package name
+  Environment: # global options 
+    port: 8000
+    host: 'localhost'
+    name: 'Eric'
+  Applications:
+    - Name: 'hello-server'
+      Environment: 
+        script: 'example/example_app.py {conf}'
+    
+      Nodes:
+        localhost: 1

example/example_app.py

+#!/usr/bin/env python
+
+import cherrypy
+import argparse
+import yaml
+
+from pprint import pprint
+
+
+class Root(object):
+    def __init__(self, conf):
+        self.conf = conf
+
+    @cherrypy.expose
+    @cherrypy.tools.json_out()
+    def index(self):
+        return self.conf
+
+
+def run():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('conf')
+
+    args = parser.parse_args()
+
+    print 'here'
+    config_file = open(args.conf, 'r')
+    conf = yaml.load(config_file)
+    pprint(conf)
+    cherrypy.config.update({
+        'server.socket_host': conf['host'],
+        'server.socket_port': conf['port'],
+    })
+
+
+    cherrypy.tree.mount(Root(conf), '/')
+    cherrypy.engine.start()
+    cherrypy.engine.block()
+    
+if __name__ == '__main__':
+    run()
+        
+from paver.easy import *
+from paver.setuputils import setup
+
+
+install_requirements = [
+    'cherrypy>=3.2',
+    'pyyaml',
+]
+
+setup(
+    name='Dad',
+    version='0.1',
+    author='Eric Larson',
+    author_email='eric@ionrock.org',
+    packages=['dad'],
+    install_requires=install_requirements,
+    entry_points={
+        'console_scripts': [
+            'dad.web = dad.web.control.root:run',
+        ]
+    }
+)
+
+@task
+def web():
+    from pkg_resources import load_entry_point
+    dad_web = load_entry_point('dad', 'console_scripts', 'dad.web')
+    dad_web(['example/example.yaml'])