Commits

Robert Brewer  committed 238fba6

Merged the reexec branch. Autoreload now uses exec instead of spawn, and therefore never runs more than one process at a time. There's a new test for autoreload in test_states.py.

The state-management for the HTTP server is a mess, by the way. I only got test_states' KeyboardInterrupt test working by inlining the client threads. So wait() and interrupts, etc. only really work when there's a single master thread. This needs fixed.

  • Participants
  • Parent commits 6d31a77
  • Branches cherrypy

Comments (0)

Files changed (6)

File _cpengine.py

 """Create and manage the CherryPy application engine."""
 
 import cgi
+import os
 import sys
 import threading
 import time
 
 import cherrypy
 from cherrypy import _cprequest
-from cherrypy.lib import autoreload
 
 # Use a flag to indicate the state of the application engine.
 STOPPED = 0
 STARTED = 1
 
 
+def fileattr(m):
+    if hasattr(m, "__loader__"):
+        if hasattr(m.__loader__, "archive"):
+            return m.__loader__.archive
+    return getattr(m, "__file__", None)
+
+
 class Engine(object):
     """The application engine, which exposes a request interface to (HTTP) servers."""
     
     
     def __init__(self):
         self.state = STOPPED
-        self.interrupt = None
         
         # Startup/shutdown hooks
         self.on_start_engine_list = []
         self.on_start_thread_list = []
         self.on_stop_thread_list = []
         self.seen_threads = {}
+        
+        self.mtimes = {}
+        self.reload_files = []
     
     def start(self, blocking=True):
         """Start the application engine."""
         self.state = STARTING
-        self.interrupt = None
         
         conf = cherrypy.config.get
         
         if conf("log_config", True):
             cherrypy.config.log_config()
         
-        # Autoreload. Note that, if we're not starting our own HTTP server,
-        # autoreload could do Very Bad Things when it calls sys.exit, but
-        # deployers will just have to be educated and responsible for it.
-        if conf('autoreload.on', False):
-            try:
-                freq = conf('autoreload.frequency', 1)
-                autoreload.main(self._start, args=(blocking,), freq=freq)
-            except KeyboardInterrupt:
-                cherrypy.log("<Ctrl-C> hit: shutting down autoreloader", "ENGINE")
-                cherrypy.server.stop()
-                self.stop()
-            except SystemExit:
-                cherrypy.log("SystemExit raised: shutting down autoreloader", "ENGINE")
-                cherrypy.server.stop()
-                self.stop()
-                # We must raise here: if this is a process spawned by
-                # autoreload, then it must return its error code to
-                # the parent.
-                raise
-            return
-        
-        self._start(blocking)
-    
-    def _start(self, blocking=True):
-        # This is in a separate function so autoreload can call it.
         for func in self.on_start_engine_list:
             func()
         self.state = STARTED
     def block(self):
         """Block forever (wait for stop(), KeyboardInterrupt or SystemExit)."""
         try:
+            autoreload = cherrypy.config.get('autoreload.on', False)
+            if autoreload:
+                i = 0
+                freq = cherrypy.config.get('autoreload.frequency', 1)
+            
             while self.state != STOPPED:
                 time.sleep(.1)
-                if self.interrupt:
-                    raise self.interrupt
+                
+                # Autoreload
+                if autoreload:
+                    i += .1
+                    if i > freq:
+                        i = 0
+                        self.autoreload()
         except KeyboardInterrupt:
             cherrypy.log("<Ctrl-C> hit: shutting down app engine", "ENGINE")
             cherrypy.server.stop()
             raise
         except:
             # Don't bother logging, since we're going to re-raise.
-            self.interrupt = sys.exc_info()[1]
             # Note that we don't stop the HTTP server here.
             self.stop()
             raise
     
+    def reexec(self):
+        """Re-execute the current process."""
+        cherrypy.server.stop()
+        self.stop()
+        
+        args = sys.argv[:]
+        cherrypy.log("Re-spawning %s" % " ".join(args), "ENGINE")
+        args.insert(0, sys.executable)
+        
+        if sys.platform == "win32":
+            args = ['"%s"' % arg for arg in args]
+        os.execv(sys.executable, args)
+    
+    def autoreload(self):
+        """Reload the process if registered files have been modified."""
+        for filename in map(fileattr, sys.modules.values()) + self.reload_files:
+            if filename:
+                if filename.endswith(".pyc"):
+                    filename = filename[:-1]
+                
+                try:
+                    mtime = os.stat(filename).st_mtime
+                except OSError:
+                    if filename in self.mtimes:
+                        # The file was probably deleted.
+                        self.reexec()
+                
+                if filename not in self.mtimes:
+                    self.mtimes[filename] = mtime
+                    continue
+                
+                if mtime > self.mtimes[filename]:
+                    # The file has been modified.
+                    self.reexec()
+    
     def stop(self):
         """Stop the application engine."""
         if self.state != STOPPED:
         """Block the caller until ready to receive requests (or error)."""
         while not self.ready:
             time.sleep(.1)
-            if self.interrupt:
-                raise self.interrupt
     
     def _is_ready(self):
         return bool(self.state == STARTED)
     def __init__(self, msg):
         self.msg = msg
     
+    def close(self):
+        pass
+    
     def run(self, request_line, headers, rfile):
         self.method = "GET"
         cherrypy.HTTPError(503, self.msg).set_response()

File _cpserver.py

     
     def start(self, server=None):
         """Main function. MUST be called from the main thread."""
+        self.interrupt = None
+        
         conf = cherrypy.config.get
         if server is None:
             server = conf('server.instance', None)
     
     def wait(self):
         """Wait until the HTTP server is ready to receive requests."""
-        while (not getattr(self.httpserver, "ready", True)
+        while (not getattr(self.httpserver, "ready", False)
                and not self.interrupt):
             time.sleep(.1)
+        if self.interrupt:
+            raise self.interrupt
         
         # Wait for port to be occupied
         if cherrypy.config.get('server.socket_port'):
         else:
             # httpstop() MUST block until the server is *truly* stopped.
             httpstop()
+            conf = cherrypy.config.get
+            if conf('server.socket_port'):
+                host = conf('server.socket_host')
+                port = conf('server.socket_port')
+                wait_for_free_port(host, port)
             cherrypy.log("HTTP Server shut down", "HTTP")
     
     def restart(self):

File _cpwsgiserver.py

     
     version = "CherryPy/3.0.0alpha"
     ready = False
-    interrupt = None
+    _interrupt = None
     RequestHandlerClass = HTTPRequest
     
     def __init__(self, bind_addr, wsgi_app, numthreads=10, server_name=None,
         while self.ready:
             self.tick()
             if self.interrupt:
+                while self.interrupt is True:
+                    # Wait for self.stop() to complete
+                    time.sleep(0.1)
                 raise self.interrupt
     
     def tick(self):
         try:
             s, addr = self.socket.accept()
+            if not self.ready:
+                return
             if hasattr(s, 'settimeout'):
                 s.settimeout(self.timeout)
             request = self.RequestHandlerClass(s, addr, self)
             # notice keyboard interrupts on Win32, which don't interrupt
             # accept() by default
             return
+        except socket.error, x:
+            if x.args[1] == "Bad file descriptor":
+                # Our socket was closed
+                return
+            raise
+    
+    def _get_interrupt(self):
+        return self._interrupt
+    def _set_interrupt(self, interrupt):
+        self._interrupt = True
+        self.stop()
+        self._interrupt = interrupt
+    interrupt = property(_get_interrupt, _set_interrupt)
     
     def stop(self):
         """Gracefully shutdown a server that is serving forever."""
         self.ready = False
-        s = getattr(self, "socket", None)
-        if s and hasattr(s, "close"):
-            s.close()
+        
+        sock = getattr(self, "socket", None)
+        if sock:
+            if not isinstance(self.bind_addr, basestring):
+                # Ping our own socket to make accept() return immediately.
+                try:
+                    host, port = sock.getsockname()[:2]
+                except socket.error, x:
+                    if x.args[1] != "Bad file descriptor":
+                        raise
+                else:
+                    for res in socket.getaddrinfo(host, port, socket.AF_UNSPEC,
+                                                  socket.SOCK_STREAM):
+                        af, socktype, proto, canonname, sa = res
+                        s = None
+                        try:
+                            s = socket.socket(af, socktype, proto)
+                            # See http://groups.google.com/group/cherrypy-users/
+                            #        browse_frm/thread/bbfe5eb39c904fe0
+                            s.settimeout(1.0)
+                            s.connect((host, port))
+                            s.close()
+                        except socket.error:
+                            if s:
+                                s.close()
+            if hasattr(sock, "close"):
+                sock.close()
+            self.socket = None
         
         # Must shut down threads here so the code that calls
         # this method can know when all threads are stopped.
         
         # Don't join currentThread (when stop is called inside a request).
         current = threading.currentThread()
-        for worker in self._workerThreads:
+        while self._workerThreads:
+            worker = self._workerThreads.pop()
             if worker is not current and worker.isAlive:
-                worker.join()
-        
-        self._workerThreads = []
+                try:
+                    worker.join()
+                except AssertionError:
+                    pass
+
 
 import cherrypy
 
+
 environments = {
     "development": {
         'autoreload.on': True,
 def merge(base, other):
     """Merge one app config (from a dict, file, or filename) into another."""
     if isinstance(other, basestring):
-        if other not in cherrypy.lib.autoreload.reloadFiles:
-            cherrypy.lib.autoreload.reloadFiles.append(other)
+        if other not in cherrypy.engine.reload_files:
+            cherrypy.engine.reload_files.append(other)
         other = Parser().dict_from_file(other)
     elif hasattr(other, 'read'):
         other = Parser().dict_from_file(other)
 def update(conf):
     """Update globalconf from a dict, file or filename."""
     if isinstance(conf, basestring):
-        if conf not in cherrypy.lib.autoreload.reloadFiles:
-            cherrypy.lib.autoreload.reloadFiles.append(conf)
+        if conf not in cherrypy.engine.reload_files:
+            cherrypy.engine.reload_files.append(conf)
         conf = Parser().dict_from_file(conf)
     elif hasattr(conf, 'read'):
         conf = Parser().dict_from_file(conf)

File lib/autoreload.py

-# autoreloading launcher
-# stolen a lot from Ian Bicking's WSGIKit (www.wsgikit.org)
-
-import os
-import sys
-import time
-import thread
-
-RUN_RELOADER = True
-reloadFiles = []
-
-def reloader_thread(freq):
-    mtimes = {}
-    
-    def fileattr(m):
-        if hasattr(m, "__loader__"):
-            if hasattr(m.__loader__, "archive"):
-                return m.__loader__.archive
-        return getattr(m, "__file__", None)
-    
-    while RUN_RELOADER:
-        for filename in map(fileattr, sys.modules.values()) + reloadFiles:
-            if filename:
-                if filename.endswith(".pyc"):
-                    filename = filename[:-1]
-                try:
-                    mtime = os.stat(filename).st_mtime
-                except OSError:
-                    sys.exit(3) # force reload
-                if filename not in mtimes:
-                    mtimes[filename] = mtime
-                    continue
-                if mtime > mtimes[filename]:
-                    sys.exit(3) # force reload
-        time.sleep(freq)
-
-def restart_with_reloader():
-    while True:
-        args = [sys.executable] + sys.argv
-        if sys.platform == "win32":
-            args = ['"%s"' % arg for arg in args]
-        new_environ = os.environ.copy()
-        new_environ["RUN_MAIN"] = 'true'
-        exit_code = os.spawnve(os.P_WAIT, sys.executable, args, new_environ)
-        if exit_code != 3:
-            return exit_code
-
-def main(main_func, args=None, kwargs=None, freq=1):
-    if os.environ.get("RUN_MAIN") == "true":
-        
-        if args is None:
-            args = ()
-        if kwargs is None:
-            kwargs = {}
-        thread.start_new_thread(main_func, args, kwargs)
-        
-        # If KeyboardInterrupt is raised within reloader_thread,
-        # let it propagate out to the caller.
-        reloader_thread(freq)
-    else:
-        # If KeyboardInterrupt is raised within restart_with_reloader,
-        # let it propagate out to the caller.
-        sys.exit(restart_with_reloader())

File test/test_states.py

+import os
+import sys
+import threading
+import time
+
 import test
 test.prefer_parent_path()
 
-import threading
-
 import cherrypy
 
 
         if self.server_class:
             
             # Raise a keyboard interrupt in the HTTP server's main thread.
-            def interrupt():
-                cherrypy.server.wait()
-                cherrypy.server.httpserver.interrupt = KeyboardInterrupt
-            threading.Thread(target=interrupt).start()
+            # We must start the server in this, the main thread
+            cherrypy.engine.start(blocking=False)
+            cherrypy.server.start(self.server_class)
+            cherrypy.server.httpserver.interrupt = KeyboardInterrupt
+            while cherrypy.engine.state != 0:
+                time.sleep(0.1)
             
-            # We must start the server in this, the main thread
-            cherrypy.server.start(self.server_class)
-            # Time passes...
             self.assertEqual(db_connection.running, False)
             self.assertEqual(len(db_connection.threads), 0)
+            self.assertEqual(cherrypy.engine.state, 0)
             
             # Raise a keyboard interrupt in a page handler; on multithreaded
             # servers, this should occur in one of the worker threads.
             # This should raise a BadStatusLine error, since the worker
             # thread will just die without writing a response.
-            def interrupt():
-                cherrypy.server.wait()
-                from httplib import BadStatusLine
-                self.assertRaises(BadStatusLine, self.getPage, "/ctrlc")
-            threading.Thread(target=interrupt).start()
+            cherrypy.engine.start(blocking=False)
+            cherrypy.server.start(self.server_class)
             
-            cherrypy.server.start(self.server_class)
-            # Time passes...
+            from httplib import BadStatusLine
+            try:
+                self.getPage("/ctrlc")
+            except BadStatusLine:
+                pass
+            else:
+                print self.body
+                self.fail("AssertionError: BadStatusLine not raised")
+            
+            while cherrypy.engine.state != 0:
+                time.sleep(0.1)
             self.assertEqual(db_connection.running, False)
             self.assertEqual(len(db_connection.threads), 0)
+    
+    def test_3_Autoreload(self):
+        if self.server_class:
+            demoscript = os.path.join(os.getcwd(), os.path.dirname(__file__),
+                                      "test_states_demo.py")
+            
+            # Start the demo script in a new process
+            host = cherrypy.config.get("server.socket_host")
+            port = cherrypy.config.get("server.socket_port")
+            cherrypy._cpserver.wait_for_free_port(host, port)
+            os.spawnl(os.P_NOWAIT, sys.executable, sys.executable,
+                      demoscript, host, str(port))
+            cherrypy._cpserver.wait_for_occupied_port(host, port)
+            
+            try:
+                self.getPage("/pid")
+                pid = self.body
+                
+                # Give the autoreloader time to cache the file time.
+                time.sleep(2)
+                
+                # Touch the file
+                f = open(demoscript, 'ab')
+                f.write(" ")
+                f.close()
+                
+                # Give the autoreloader time to re-exec the process
+                time.sleep(2)
+                cherrypy._cpserver.wait_for_occupied_port(host, port)
+                
+                self.getPage("/pid")
+                self.assertNotEqual(self.body, pid)
+            finally:
+                # Shut down the spawned process
+                self.getPage("/stop")
 
 
 db_connection = None
         cherrypy.engine.on_start_thread_list.append(db_connection.startthread)
         cherrypy.engine.on_stop_thread_list.append(db_connection.stopthread)
         
-        helper.CPTestRunner.run(suite)
+        import pyconquer
+        tr = pyconquer.Logger("cherrypy")
+        tr.out = open(os.path.join(os.path.dirname(__file__), "state.log"), "wb")
+        try:
+            tr.start()
+            helper.CPTestRunner.run(suite)
+        finally:
+            tr.stop()
+            tr.out.close()
     finally:
-        cherrypy.server.stop()
+        if cherrypy.server.httpserver.ready:
+            cherrypy.server.stop()
         cherrypy.engine.stop()