Commits

Robert Brewer committed adb12ff

New deadlock monitor which sets Response.timed_out to True if Response.time < now - config.get("deadlock_timeout"). The request thread periodically checkes Response.timed_out and raises TimeoutError if it is True. Current checks are in HookMap.run, Request.respond, Body.__set__, and Response.finalize, more can be added later if needed.

  • Participants
  • Parent commits 40100fe
  • Branches cherrypy

Comments (0)

Files changed (6)

 
 import logging as _logging
 
-from _cperror import HTTPError, HTTPRedirect, InternalRedirect, NotFound, WrongConfigValue
+from _cperror import HTTPError, HTTPRedirect, InternalRedirect, NotFound
+from _cperror import WrongConfigValue, TimeoutError
 import config
 
 import _cptools
         self.on_stop_thread_list = []
         self.seen_threads = {}
         
+        self.servings = []
+        
         self.mtimes = {}
         self.reload_files = []
     
         
         for func in self.on_start_engine_list:
             func()
+        
         self.state = STARTED
+        
+        freq = float(cherrypy.config.get('deadlock_poll_freq', 60))
+        if freq > 0:
+            self.monitor_thread = threading.Timer(freq, self.monitor)
+            self.monitor_thread.start()
+        
         if blocking:
             self.block()
     
             for func in self.on_stop_engine_list:
                 func()
             
+            if self.monitor_thread:
+                self.monitor_thread.cancel()
+                self.monitor_thread = None
+            
             self.state = STOPPED
             cherrypy.log("CherryPy shut down", "ENGINE")
     
         scheme: either "http" or "https"; defaults to "http"
         """
         if self.state == STOPPED:
-            r = NotReadyRequest("The CherryPy engine has stopped.")
+            req = NotReadyRequest("The CherryPy engine has stopped.")
         elif self.state == STARTING:
-            r = NotReadyRequest("The CherryPy engine could not start.")
+            req = NotReadyRequest("The CherryPy engine could not start.")
         else:
             # Only run on_start_thread_list if the engine is running.
             threadID = threading._get_ident()
                 
                 for func in self.on_start_thread_list:
                     func(i)
-            r = self.request_class(local_host, remote_host, scheme)
-        cherrypy.serving.request = r
-        cherrypy.serving.response = self.response_class()
-        return r
+            req = self.request_class(local_host, remote_host, scheme)
+        cherrypy.serving.request = req
+        cherrypy.serving.response = resp = self.response_class()
+        self.servings.append((req, resp))
+        return req
+    
+    def monitor(self):
+        """Check timeout on all responses."""
+        if self.state == STARTED:
+            for req, resp in self.servings:
+                resp.check_timeout()
+            freq = float(cherrypy.config.get('deadlock_poll_freq', 60))
+            self.monitor_thread = threading.Timer(freq, self.monitor)
+            self.monitor_thread.start()
     
     def start_with_callback(self, func, args=None, kwargs=None):
         """Start, then callback the given func in a new thread."""
             import cherrypy
             path = cherrypy.request.path
         self.args = (path,)
-        HTTPError.__init__(self, 404, "The path %s was not found." % repr(path))
+        HTTPError.__init__(self, 404, "The path %r was not found." % path)
+
+
+class TimeoutError(Exception):
+    """Exception raised when Response.timed_out is detected."""
+    pass
 
 
 _HTTPErrorTemplate = '''<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
     
     def run(self, point):
         """Execute all registered callbacks for the given point."""
+        if cherrypy.response.timed_out:
+            raise cherrypy.TimeoutError()
+        
         failsafe = point in self.failsafe
         for callback in self.callbacks[point]:
             # Some hookpoints guarantee all callbacks are run even if
         if not self.closed:
             self.closed = True
             self.hooks.run('on_end_request')
+            
+            s = (self, cherrypy.serving.response)
+            try:
+                cherrypy.engine.servings.remove(s)
+            except ValueError:
+                pass
+            
             cherrypy.serving.__dict__.clear()
     
     def run(self, method, path, query_string, protocol, headers, rfile):
                     self.redirections.append(pi)
         except (KeyboardInterrupt, SystemExit):
             raise
+        except cherrypy.TimeoutError:
+            raise
         except:
             if cherrypy.config.get("throw_errors", False):
                 raise
         """Generate a response for the resource at self.path_info."""
         try:
             try:
+                if cherrypy.response.timed_out:
+                    raise cherrypy.TimeoutError()
+                
                 self.hooks = HookMap(self.hookpoints)
                 self.hooks.failsafe = ['on_start_resource', 'on_end_resource',
                                        'on_end_request']
             return obj._body
     
     def __set__(self, obj, value):
+        if cherrypy.response.timed_out:
+            raise cherrypy.TimeoutError()
         # Convert the given value to an iterable object.
         if isinstance(value, types.FileType):
             value = fileGenerator(value)
     """An HTTP Response."""
     
     # Class attributes for dev-time introspection.
-    status = None
-    header_list = None
+    status = ""
+    header_list = []
     headers = http.HeaderMap()
     simple_cookie = Cookie.SimpleCookie()
     body = Body()
+    time = None
+    timed_out = False
     
     def __init__(self):
         self.status = None
         self.header_list = None
-        self.body = None
+        self._body = []
         self.time = time.time()
         
         self.headers = http.HeaderMap()
     
     def finalize(self):
         """Transform headers (and cookies) into cherrypy.response.header_list."""
+        if self.timed_out:
+            raise cherrypy.TimeoutError()
         
         try:
             code, reason, _ = http.validStatus(self.status)
             for line in cookie.split("\n"):
                 name, value = line.split(": ", 1)
                 h.append((name, value))
+    
+    def check_timeout(self):
+        """If now > self.time + deadlock_timeout, set self.timed_out.
+        
+        This purposefully sets a flag, rather than raising an error,
+        so that a monitor thread can interrupt the Response thread.
+        """
+        timeout = float(cherrypy.config.get('deadlock_timeout', 300))
+        if time.time() > self.time + timeout:
+            self.timed_out = True
+

test/test_states.py

         cherrypy.engine.restart()
         return "app was restarted succesfully"
     restart.exposed = True
+    
+    def block_explicit(self):
+        while True:
+            if cherrypy.response.timed_out:
+                cherrypy.response.timed_out = False
+                return "broken!"
+            time.sleep(0.1)
+    block_explicit.exposed = True
+    
+    def block_implicit(self):
+        raise cherrypy.InternalRedirect("/block_implicit")
+    block_implicit.exposed = True
+    block_implicit._cp_config = {'recursive_redirect': True}
 
 cherrypy.tree.mount(Root())
 cherrypy.config.update({
     'log_to_screen': False,
     'environment': 'production',
+    'deadlock_poll_freq': 1,
+    'deadlock_timeout': 2,
     })
 
 class Dependency:
             self.assertEqual(db_connection.running, False)
             self.assertEqual(len(db_connection.threads), 0)
     
-    def test_3_Autoreload(self):
+    def test_3_Deadlocks(self):
+        cherrypy.engine.start(blocking=False)
+        cherrypy.server.start()
+        try:
+            self.assertNotEqual(cherrypy.engine.monitor_thread, None)
+            
+            # Request a "normal" page.
+            self.assertEqual(cherrypy.engine.servings, [])
+            self.getPage("/")
+            self.assertBody("Hello World")
+            # request.close is called async.
+            while cherrypy.engine.servings:
+                time.sleep(0.1)
+            
+            # Request a page that explicitly checks itself for deadlock.
+            # The deadlock_timeout should be 2 secs.
+            self.getPage("/block_explicit")
+            self.assertBody("broken!")
+            
+            # Request a page that implicitly breaks deadlock.
+            # If we deadlock, we want to touch as little code as possible,
+            # so we won't even call handle_error, just bail ASAP.
+            self.getPage("/block_implicit")
+            self.assertStatus(500)
+            self.assertInBody("raise cherrypy.TimeoutError()")
+        finally:
+            cherrypy.engine.stop()
+            cherrypy.server.stop()
+    
+    def test_4_Autoreload(self):
         if self.server_class:
             demoscript = os.path.join(os.getcwd(), os.path.dirname(__file__),
                                       "test_states_demo.py")

test/test_states_demo.py

                             "environment": "development",
                             })
     cherrypy.quickstart(Root())
-    
+