1. Ross Lagerwall
  2. SlaveDriver

Commits

Ross Lagerwall  committed 1463225

Refactored http_server to infoserver - it is part of the event loop now rather than running on a separate thread, along with many other changes.

Some of the changes:
Slave can now return build data i.e. rev author, description, date, etc.
Improved appearance of infoserver pages (added built-in templates).
Improved timing and return value handling.

  • Participants
  • Parent commits bd24ee3
  • Branches default

Comments (0)

Files changed (6)

File conf

View file
  • Ignore whitespace
 host = localhost
 port = 9999
 name = coolslave
-command = pull
+command = sleep1
+
+[sleep1]
+command = sleep 10
+success = builddata
+
+[sleep2]
+command = sleep 20
 
 [pull]
 command = hg pull
 
 [update]
 command = hg update -C <data>
-success = configure
+success = builddata
+
+[builddata]
+Revision Author = hg log -r <data> --template "{author|person}"
+Revision Description = hg log -r <data> --template "{desc}"
+Revision Date = hg log -r <data> --template "{date|isodate}"
+success = sleep2
 
 [configure]
 command = ./configure
 fail = clean
 
 [make]
-command = make
+command = make -j 4
 success = clean
 fail = clean
 

File http_server.py

  • Ignore whitespace
-import threading
-import http.server
-import socketserver
-
-STOPPED = 0
-BUILDING = 1
-
-STOP = -123456789
-BUILD = 1
-EXIT = 2
-
-CMDSTART = 0
-CMDFINISH = 1
-FINISHED = 2
-STDOUT = 3
-STDERR = 4
-
-class HTTPServer(socketserver.TCPServer):
-    allow_reuse_address = True
-
-class HTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
-
-    def handle_main(self, path):
-        return ''
-
-    def handle_build(self, path):
-        data = "<html><head><title>%s - Build %s</title></head><body>\n<h2>%s - Build %s</h2>" % (path[0], path[1], path[0], path[1])
-        p = int(path[1])
-        h = handlers[path[0]]
-        data += "<p>State: " + h.builds[p]['state'] + "</p>\n"
-        data += "<h4>Commands:</h4>\n\n"
-        for i, c in enumerate(h.builds[p]['cmds']):
-            data += "<h5>Command: " + c['name'] + "</h5>\n"
-            if 'rv' in c:
-                data += "Return Value: " + c['rv'] + "<br />\nExecution Time: " + c['exectime'] + "<br />\n"
-            else:
-                data += "Executing...\n"
-            data += '<a href="/%s/%s/%d">Stdout</a><br />' % (path[0], path[1], 2*i)
-            data += '<a href="/%s/%s/%d">Stderr</a><br />' % (path[0], path[1], 2*i + 1)
-
-        data += "</body></html>"
-
-        self.send_response(200)
-        self.send_header("Content-length", str(len(data)))
-        self.send_header("Content-type", "text/html")
-        self.end_headers()
-        self.wfile.write(data.encode())
-
-    def handle_data(self, path):
-        p = int(path[2])
-        pp = p // 2
-        h = handlers[path[0]]
-        if p % 2 == 0:
-            data = h.builds[int(path[1])]['cmds'][pp][STDOUT]
-        else:
-            data = h.builds[int(path[1])]['cmds'][pp][STDERR]
-
-        self.send_response(200)
-        self.send_header("Content-length", str(len(data)))
-        self.send_header("Content-type", "text/plain")
-        self.end_headers()
-        self.wfile.write(data)
-        
-
-    def handle_slave(self, path):
-        data = "<html><head><title>%s</title></head><body>\n<h2>%s</h2>" % (path[0], path[0])
-        data += "<p>Connection Status: " + ('offline' if handlers[path[0]].sock == None else 'online') + "</p>"
-        for i, b in reversed(list(enumerate(handlers[path[0]].builds))):
-            data += '<a href="/%s/%d">Build %d</a> - %s<br />' % (path[0], i, i, b['state'])
-
-        self.send_response(200)
-        self.send_header("Content-length", str(len(data)))
-        self.send_header("Content-type", "text/html")
-        self.end_headers()
-        self.wfile.write(data.encode())
-
-    def do_GET(self):
-
-        path = self.path
-        if path[0] == '/':
-            path = path[1:]
-        if path[-1] == '/':
-            path = path[:-1]
-        path = path.split('/')
-        if path == ['']:
-            self.handle_main()
-        elif len(path) == 1:
-            self.handle_slave(path)
-        elif len(path) == 2:
-            self.handle_build(path)
-        elif len(path) == 3:
-            self.handle_data(path)
-        else:
-            self.handle_error()
-
-def start(h):
-
-    global handlers
-    handlers = h
-
-    httpd = HTTPServer(("", 8000), HTTPRequestHandler)
-    httpd_thread = threading.Thread(target=httpd.serve_forever)
-    httpd_thread.daemon = True
-    httpd_thread.start()

File infoserver.py

View file
  • Ignore whitespace
+import http.server
+import string
+import time
+
+STOPPED = 0
+BUILDING = 1
+
+STOP = -123456789
+BUILD = 1
+EXIT = 2
+
+CMDSTART = 0
+CMDFINISH = 1
+FINISHED = 2
+STDOUT = 3
+STDERR = 4
+BUILDDATA = 5
+
+BUILDPAGE = string.Template("""\
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8" />
+<title>$slave - Build $buildno</title>
+<style>
+body {
+    font-size: 12px;
+    font-family: "Ubuntu", sans-serif;
+    background-color: #e7e7e7;
+    color: #464646;
+}
+td {
+    text-align: center;
+    width: 20%;
+}
+.medium {
+    font-size: 14px;
+    font-weight: bold;
+}
+</style>
+</head>
+<body>
+    <h2>$slave - Build $buildno</h2>
+    <span class="medium">State: <span style="background-color: $color">$state</span></span><br />
+    <span class="medium">Started at: $stime</span><br />
+    <span class="medium">Finished at: $ftime</span><br />
+    $builddata
+    <table style="width: 80%; margin: auto;">
+        <tr><td>Command</td><td>Time</td><td>Status</td><td>Stdout</td><td>Stderr</td></tr>
+        $commands
+    </table>
+</body>
+</html>
+""")
+BUILDPAGE_ROW = string.Template("""\
+       <tr><td style="background-color: $color">$cmd</td><td>$exectime</td><td>$status</td><td>$stdout</td><td>$stderr</td></tr>
+""")
+
+SLAVEPAGE = string.Template("""\
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8" />
+<title>$slave</title>
+<style>
+body {
+    font-size: 12px;
+    font-family: "Ubuntu", sans-serif;
+    background-color: #e7e7e7;
+    color: #464646;
+}
+td {
+    text-align: center;
+    width: 20%;
+}
+.medium {
+    font-size: 14px;
+    font-weight: bold;
+}
+</style>
+</head>
+<body>
+    <h2>$slave</h2>
+    <span class="medium">Connection: <span style="background-color: $color">$connection</span></span><br />
+    <span class="medium"><a href="/$slave/builds">See all builds</a></span><br />
+    <table style="width: 80%; margin: auto;">
+        <tr><td>Build</td><td>Status</td><td>Start Time</td></tr>
+        $builds
+    </table>
+</body>
+</html>
+""")
+SLAVEPAGE_ROW = string.Template("""\
+       <tr><td>$build</td><td style="background-color: $color">$state</td><td>$stime</td></tr>
+""")
+
+BUILDLISTPAGE = string.Template("""\
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8" />
+<title>$slave</title>
+<style>
+body {
+    font-size: 12px;
+    font-family: "Ubuntu", sans-serif;
+    background-color: #e7e7e7;
+    color: #464646;
+}
+td {
+    text-align: center;
+    width: 20%;
+}
+.medium {
+    font-size: 14px;
+    font-weight: bold;
+}
+</style>
+</head>
+<body>
+    <h2>$slave</h2>
+    <span class="medium">Connection: <span style="background-color: $color">$connection</span></span><br />
+    <table style="width: 80%; margin: auto;">
+        <tr><td>Build</td><td>Status</td><td>Start Time</td></tr>
+        $builds
+    </table>
+</body>
+</html>
+""")
+BUILDLISTPAGE_ROW = string.Template("""\
+       <tr><td>$build</td><td style="background-color: $color">$state</td><td>$stime</td></tr>
+""")
+
+MAINPAGE = string.Template("""\
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8" />
+<title>Buildbot</title>
+<style>
+body {
+    font-size: 12px;
+    font-family: "Ubuntu", sans-serif;
+    background-color: #e7e7e7;
+    color: #464646;
+}
+td {
+    text-align: center;
+    width: 20%;
+}
+</style>
+</head>
+<body>
+    <h2>Buildbot</h2>
+    <table style="width: 80%; margin: auto;">
+        <tr><td>Slave</td><td>Connection</td><td>Last Test</td></tr>
+        $slaves
+    </table>
+</body>
+</html>
+""")
+MAINPAGE_ROW = string.Template("""\
+       <tr><td>$slave</td><td style="background-color: $color1">$connection</td><td style="background-color: $color2">$state</td></tr>
+""")
+
+class HTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
+
+    def __init__(self, request, client_address, server, slaves):
+        self.slaves = slaves
+        http.server.SimpleHTTPRequestHandler.__init__(self, request, client_address, server)
+
+    def handle_main(self):
+        data = ''   
+        for name in self.slaves.keys():
+            if self.slaves[name].sock == None:
+                connection = 'Offline'
+                color1 = '#d85300'
+            else:
+                connection = 'Online'
+                color1 = '#43a612'
+
+            if len(self.slaves[name].builds) == 0:
+                state = 'No Test'
+                color2 = '#e7e7e7'
+            elif self.slaves[name].builds[-1]['state'] == True:
+                state = 'Pass'
+                color2 = '#43a612'
+            elif self.slaves[name].builds[-1]['state'] == False:
+                state = 'Fail'
+                color2 = '#d85300'
+            else:
+                state = 'Running'
+                color2 = '#eeeb00'
+
+            data += MAINPAGE_ROW.substitute(slave='<a href="%s">%s</a>' % (name, name),color1=color1,
+                        connection=connection,color2=color2,state=state)
+
+        data = MAINPAGE.substitute(slaves=data)
+
+        self.send_response(200)
+        self.send_header("Content-length", str(len(data)))
+        self.send_header("Content-type", "text/html")
+        self.end_headers()
+        self.wfile.write(data.encode())
+
+    def handle_build(self, path):
+        buildno = int(path[1])
+        build = self.slaves[path[0]].builds[buildno]
+        data = ''   
+        for i, cmd in enumerate(build['cmds']):
+            if 'rv' in cmd:
+                exectime = round(cmd['ftime'] - cmd['stime'], 2)
+                status = cmd['rv']
+                color = '#43a612' if cmd['rv'] == '0' else '#d85300'
+            else:
+                exectime = round(time.time() - cmd['stime'], 2)
+                status = ''
+                color = '#eeeb00'
+            data += BUILDPAGE_ROW.substitute(color=color, cmd=cmd['name'],exectime=exectime,status=status,
+                        stdout='<a href="/%s/%s/%d">%d bytes</a>' % (path[0], path[1], 2*i, len(cmd[STDOUT])),
+                        stderr='<a href="/%s/%s/%d">%d bytes</a>' % (path[0], path[1], 2*i + 1, len(cmd[STDERR])))
+
+        if build['state'] == True:
+            state = 'Pass'
+            color = '#43a612'
+            stime = time.strftime('%x %X', time.localtime(build['stime']))
+            ftime = time.strftime('%x %X', time.localtime(build['ftime']))
+        elif build['state'] == False:
+            state = 'Fail'
+            color = '#d85300'
+            stime = time.strftime('%x %X', time.localtime(build['stime']))
+            ftime = time.strftime('%x %X', time.localtime(build['ftime']))
+        else:
+            state = 'Running'
+            color = '#eeeb00'
+            stime = time.strftime('%x %X', time.localtime(build['stime']))
+            ftime = ''
+
+        print(build['job'])
+        builddata = ''
+        for k in build['job'].keys():
+            builddata += '<span class="medium">%s: %s</span><br />' % (k, build['job'][k])
+        data = BUILDPAGE.substitute(color=color, slave=path[0], buildno=path[1], state=state, commands=data,
+                            stime=stime,ftime=ftime, builddata=builddata)
+
+        self.send_response(200)
+        self.send_header("Content-length", str(len(data)))
+        self.send_header("Content-type", "text/html")
+        self.end_headers()
+        self.wfile.write(data.encode())
+
+    def handle_build_list(self, path):
+        slave = self.slaves[path[0]]
+        data = ''
+        for i, b in reversed(list(enumerate(self.slaves[path[0]].builds))):
+            if b['state'] == True:
+                state = 'Pass'
+                color = '#43a612'
+            elif b['state'] == False:
+                state = 'Fail'
+                color = '#d85300'
+            else:
+                state = 'Running'
+                color = '#eeeb00'
+            stime = time.strftime('%x %X', time.localtime(b['stime']))
+            data += BUILDLISTPAGE_ROW.substitute(build='<a href="/%s/%d">Build %d</a>' % (path[0], i, i),
+                        state=state, color=color,stime=stime)
+
+        if slave.sock == None:
+            connection = 'Offline'
+            color = '#d85300'
+        else:
+            connection = 'Online'
+            color = '#43a612'
+
+        data = BUILDLISTPAGE.substitute(color=color, slave=path[0], connection=connection, builds=data)
+
+        self.send_response(200)
+        self.send_header("Content-length", str(len(data)))
+        self.send_header("Content-type", "text/html")
+        self.end_headers()
+        self.wfile.write(data.encode())
+
+    def handle_data(self, path):
+        p = int(path[2])
+        pp = p // 2
+        h = self.slaves[path[0]]
+        if p % 2 == 0:
+            data = h.builds[int(path[1])]['cmds'][pp][STDOUT]
+        else:
+            data = h.builds[int(path[1])]['cmds'][pp][STDERR]
+
+        self.send_response(200)
+        self.send_header("Content-length", str(len(data)))
+        self.send_header("Content-type", "text/plain")
+        self.end_headers()
+        self.wfile.write(data)
+        
+
+    def handle_slave(self, path):
+        slave = self.slaves[path[0]]
+        data = ''
+        for i, b in reversed(list(enumerate(self.slaves[path[0]].builds))[-5:]):
+            if b['state'] == True:
+                state = 'Pass'
+                color = '#43a612'
+            elif b['state'] == False:
+                state = 'Fail'
+                color = '#d85300'
+            else:
+                state = 'Running'
+                color = '#eeeb00'
+            stime = time.strftime('%x %X', time.localtime(b['stime']))
+            data += SLAVEPAGE_ROW.substitute(build='<a href="/%s/%d">Build %d</a>' % (path[0], i, i),
+                        state=state, color=color,stime=stime)
+
+        if slave.sock == None:
+            connection = 'Offline'
+            color = '#d85300'
+        else:
+            connection = 'Online'
+            color = '#43a612'
+
+        data = SLAVEPAGE.substitute(color=color, slave=path[0], connection=connection, builds=data)
+
+        self.send_response(200)
+        self.send_header("Content-length", str(len(data)))
+        self.send_header("Content-type", "text/html")
+        self.end_headers()
+        self.wfile.write(data.encode())
+
+    def handle_error(self):
+        data = "<html><head><title>Error</title></head><body>Error</body></html>"
+
+        self.send_response(404)
+        self.send_header("Content-length", str(len(data)))
+        self.send_header("Content-type", "text/html")
+        self.end_headers()
+        self.wfile.write(data.encode())
+
+    def do_GET(self):
+        path = self.path
+        if path[0] == '/':
+            path = path[1:]
+        if len(path) > 0 and path[-1] == '/':
+            path = path[:-1]
+        path = path.split('/')
+        if path == ['']:
+            self.handle_main()
+        elif len(path) == 1 and path[0] not in self.slaves:
+            self.handle_error()
+        elif len(path) == 1:
+            self.handle_slave(path)
+        elif len(path) == 2 and path[1] == 'builds':
+            self.handle_build_list(path)
+        elif len(path) == 2:
+            self.handle_build(path)
+        elif len(path) == 3:
+            self.handle_data(path)
+        else:
+            self.handle_error()

File master.py

View file
  • Ignore whitespace
 import select
 import socket
 import configparser
-import http_server
+import infoserver
 import slavehandler
 import struct
 
 el.bind(('', 9998))
 el.listen(5)
 
-readers = [s, el]
-handlers = {}
+httpd = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+httpd.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+httpd.bind(('', 8000))
+httpd.listen(5)
+
+readers = [s, el, httpd]
+slaves = {}
 
 conf = configparser.ConfigParser()
 conf.read('/home/ross/code/SlaveDriver/serverconf')
 
 for key in conf:
     if key != 'general' and key != 'DEFAULT':
-        print(key)
-        handlers[key] = slavehandler.SlaveHandler(key)
-
-print(handlers)
-
-http_server.start(handlers)
+        slaves[key] = slavehandler.SlaveHandler(key)
 
 while True:
 
             length = struct.unpack('i', read(conn.recv, 4))[0]
             name = read(conn.recv, length).decode()
             print(name)
-            if name not in handlers:
+            if name not in slaves:
                 conn.close()
             else:
-                handlers[name].online(conn)
+                slaves[name].online(conn)
                 readers.append(conn)
         elif c == el:
             conn, addr = el.accept()
             length = struct.unpack('i', read(conn.recv, 4))[0]
             data = read(conn.recv, length)
 
-            for h in handlers.keys():
-                handlers[h].add_job(command, data)
+            for h in slaves.keys():
+                slaves[h].add_job(command, data)
+        elif c == httpd:
+            conn, addr = httpd.accept()
+            infoserver.HTTPRequestHandler(conn, addr, None, slaves)
         else:
-            for name in handlers.keys():
-                if handlers[name].sock == c:
-                    if not handlers[name].handle():
-                        handlers[name].offline()
+            print("Enter Slavehandler")
+            for name in slaves.keys():
+                if slaves[name].sock == c:
+                    if not slaves[name].handle():
+                        slaves[name].offline()
                         readers.remove(c)
+            print("Exit Slavehandler")

File slave.py

View file
  • Ignore whitespace
 import socket
 import subprocess
 import select
-import time
 import struct
 
 STOP = -123456789
 FINISHED = 2
 STDOUT = 3
 STDERR = 4
+BUILDDATA = 5
 
 class Slave:
     def __init__(self, conf):
         self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         self.sock.connect((conf['general']['host'], int(conf['general']['port'])))
         self.rfile = self.sock.makefile('rb', 0)
-        self.wfile = self.sock.makefile('wb', 4096)
+        self.wfile = self.sock.makefile('wb', 0)
         self.write(conf['general']['name'])
         self.wfile.flush()
 
         cmd = self.conf['general']['command']
 
         while True:
-            self.write_status(CMDSTART)
-            self.write(cmd)
-            self.wfile.flush()
-            res, exectime = self.run_command(self.conf[cmd]['command'].replace('<data>', data))
-            self.write_status(CMDFINISH)
-            self.write("%d,%.2f" % (res, exectime))
-            self.wfile.flush()
-            move = 'success' if res == 0 else 'fail'
-            if move in self.conf[cmd]:
-                cmd = self.conf[cmd][move]
+            if cmd == 'builddata':
+                self.write_status(BUILDDATA)
+                self.write_status(len(self.conf[cmd]) - 1)
+                for dataitem in self.conf[cmd]:
+                    if dataitem != 'success':
+                        self.write(dataitem)
+                        self.write(subprocess.check_output(self.conf[cmd][dataitem].replace('<data>', data), shell=True))
+                cmd = self.conf[cmd]['success']
             else:
-                break
+                self.write_status(CMDSTART)
+                self.write(cmd)
+                res = self.run_command(self.conf[cmd]['command'].replace('<data>', data))
+                self.write_status(CMDFINISH)
+                self.write(str(res))
+                move = 'success' if res == 0 else 'fail'
+                if move in self.conf[cmd]:
+                    cmd = self.conf[cmd][move]
+                else:
+                    break
 
     def run(self):
         while True:
 
             self.build(data)
             self.write_status(FINISHED)
-            self.wfile.flush()
 
     def write(self, data):
         if isinstance(data, str):
         self.wfile.write(struct.pack('i', status))
 
     def run_command(self, command):
-        start_time = time.time()
         p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         readers = [p.stdout, p.stderr, self.rfile]
         while len(readers) > 1:
                     else:
                         exit(1) # something went wrong
         p.wait()
-        return p.returncode, time.time() - start_time
+        return p.returncode
 
 
 conf = configparser.ConfigParser()

File slavehandler.py

View file
  • Ignore whitespace
 import struct
+import time
 
 STOPPED = 0
 BUILDING = 1
 FINISHED = 2
 STDOUT = 3
 STDERR = 4
+BUILDDATA = 5
 
 def read(f, length):
     data = b''
 
     def online(self, sock):
         self.sock = sock
+        self.start_job()
 
     def offline(self):
         self.sock = None
         if cmd == STOP or cmd == EXIT:
             self.sock.sendall(struct.pack('ii', cmd, len(data)) + data)
         else:
-            self.jobs.append((cmd, data))
+            self.jobs.append((cmd, data, time.time()))
             if self.state == STOPPED:
                 self.start_job()
 
     def start_job(self):
-        if len(self.jobs) > 0:
-            cmd, data = self.jobs[0]
+        if len(self.jobs) > 0 and self.sock != None:
+            cmd, data, addtime = self.jobs[0]
             self.jobs = self.jobs[1:]
-            self.builds.append({'cmds':[], 'state':'Running'})
+            self.builds.append({'cmds':[], 'state':'Running', 'stime':time.time(), 'job':{}})
             self.sock.sendall(struct.pack('ii', cmd, len(data)) + data)
             self.state = BUILDING
 
         status = struct.unpack("i", raw_status)[0]
 
         if status == CMDSTART:
-            self.builds[-1]['cmds'].append({STDOUT:b'', STDERR:b'', 'name':self.read_str()})
+            self.builds[-1]['cmds'].append({STDOUT:b'', STDERR:b'', 'name':self.read_str(),
+                        'stime':time.time()})
         elif status == CMDFINISH:
-            self.builds[-1]['cmds'][-1]['rv'], self.builds[-1]['cmds'][-1]['exectime'] = self.read_str().split(',')
+            self.builds[-1]['cmds'][-1]['rv'] = self.read_str()
+            self.builds[-1]['cmds'][-1]['ftime'] = time.time()
         elif status == STDERR or status == STDOUT:
             d = self.read_data()
             self.builds[-1]['cmds'][-1][status] += d
         elif status == FINISHED:
-            self.builds[-1]['state'] = 'Finished'
+            self.builds[-1]['state'] = True
+            self.builds[-1]['ftime'] = time.time()
+            for c in self.builds[-1]['cmds']:
+                if c['rv'] != '0':
+                    self.builds[-1]['state'] = False
             self.state = STOPPED
             self.start_job()
-
+        elif status == BUILDDATA:
+            jobdata = self.builds[-1]['job']
+            n = struct.unpack('i', read(self.sock.recv, 4))[0]
+            for i in range(n):
+                key = self.read_str()
+                jobdata[key] = self.read_str()
+            print(jobdata)
         return True