Anonymous avatar Anonymous committed 7967bae

Ticket #59 - a bunch of changes:
- Uniform return type: internally, the cpg.response.body is always assumed to be an iterable. Simple strings are wrapped in a single-item list.
- All filters were modified to work with iterables & generators.
- The gzip filter is now based on the zlib module, and works as a generator.
- The encoding filter now checks for a content-type of 'text/html', and works on it.
- The xmlrpcfilter was added to the standard library of filters.

Comments (0)

Files changed (9)

cherrypy/_cphttptools.py

         _cputil.getSpecialFunction('_cpSaveSessionData')(sessionId, cpg.request.sessionMap, expirationTime)
 
     # Set the content-length
-    if cpg.response.headerMap.has_key('Content-Length') and cpg.response.headerMap['Content-Length']==0:
-        cpg.response.headerMap['Content-Length'] = len(cpg.response.body)
+    if (cpg.response.headerMap.has_key('Content-Length') and
+        cpg.response.headerMap['Content-Length']==0):
+        #body = "" 
+        #body = ''.join(list(cpg.response.body))  # a local var is more efficient here
+        #body = u''.join(cpg.response.body)  # a local var is more efficient here
+        #body = ''
+        #for line in cpg.response.body:
+        #    print body
+        #    body += line
+        buf = StringIO.StringIO()
+        [buf.write(x) for x in cpg.response.body]
+        buf.seek(0)
+        cpg.response.body = [buf.read()]
+        cpg.response.headerMap['Content-Length'] = len(cpg.response.body[0])
 
     wfile.write('%s %s\r\n' % (cpg.response.headerMap['protocolVersion'], cpg.response.headerMap['Status']))
     for key, valueList in cpg.response.headerMap.items():
         wfile.write(cookie+'\r\n')
     wfile.write('\r\n')
 
-    applyFilterList('afterResponseHeader')
-
-    applyFilterList('beforeResponseFullBody')
-
-    # Check that the response body is a string
-    if type(cpg.response.body) != types.StringType:
-        raise cperror.WrongResponseType
-
-    wfile.write(cpg.response.body)
+    for line in cpg.response.body:
+        wfile.write(line)
+    
+    # finalization hook for filter cleanup & logging purposes
+    applyFilterList('afterResponse')
 
 def handleRequest(wfile):
     # Clean up expired sessions if needed:
          
     # Remove "root" from objectPathList and join it to get objectPath
     cpg.request.objectPath = '/' + '/'.join(objectPathList[1:])
-    cpg.response.body = func(*(virtualPathList + cpg.request.paramList), **(cpg.request.paramMap))
+    body = func(*(virtualPathList + cpg.request.paramList), **(cpg.request.paramMap))
+    
+    # builds a uniform return type
+    if not isinstance(body, types.GeneratorType):
+        cpg.response.body = [body]
+    else:
+        cpg.response.body = body
 
     if cpg.response.sendResponse:
         sendResponse(wfile)

cherrypy/lib/filter/basefilter.py

         """ Called before starting to write response """
         pass
 
-    def afterResponseHeader(self):
-        """ Called after writing the response header """
+    def afterResponse(self):
+        """ Called after writing the response (header & body included) """
         pass
 
-    def beforeResponseFullBody(self):
-        """ Called before writing the full response body """
-        pass
-
-    def beforeResponseOnTheFlyBody(self):
-        """ Called before writing a bit of the reponse (only used
-            when using "on-the-fly" response.
-        """
-        pass
-

cherrypy/lib/filter/baseurlfilter.py

             newBaseUrl = cpg.request.headerMap.get("X-Forwarded-Host", self.baseUrl)
         else:
             newBaseUrl = self.baseUrl
-	if newBaseUrl.find("://") == -1:
-	    # add http:// or https:// if needed	
-	    newBaseUrl = cpg.request.base[:cpg.request.base.find("://")] + "://" + newBaseUrl
+        if newBaseUrl.find("://") == -1:
+            # add http:// or https:// if needed	
+            newBaseUrl = cpg.request.base[:cpg.request.base.find("://")] + "://" + newBaseUrl
 
         cpg.request.browserUrl = cpg.request.browserUrl.replace(
             cpg.request.base, newBaseUrl)

cherrypy/lib/filter/encodingfilter.py

     Filter that automatically encodes the response.
     """
 
-    def __init__(self, encoding = 'utf-8'):
+    def __init__(self, encoding = 'utf-8', mimeTypeList = ['text/html']):
         self.encoding = encoding
+        self.mimeTypeList = mimeTypeList
 
     def beforeResponse(self):
-        if isinstance(cpg.response.body, unicode):
-            # Encode the response
-            cpg.response.body = cpg.response.body.encode(self.encoding)
+        ct = cpg.response.headerMap.get('Content-Type').split(';')[0]
+        if (ct in self.mimeTypeList):
             # Add "charset=..." to response Content-Type header
             contentType = cpg.response.headerMap.get("Content-Type")
             if contentType and 'charset' not in contentType:
-                cpg.response.headerMap["Content-Type"] += "; charset=%s" % self.encoding
+                cpg.response.headerMap["Content-Type"] += ";charset=%s" % self.encoding
+            # Return a generator that encodes the sequence
+            cpg.response.body = self.encode_body(cpg.response.body)
 
+    def encode_body(self, body):
+        for line in body:
+            yield line.encode(self.encoding)

cherrypy/lib/filter/gzipfilter.py

 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 """
 
-import gzip, StringIO
+import zlib
+import struct
+import time
 from basefilter import BaseOutputFilter
 from cherrypy import cpg
 
     Filter that gzips the response.
     """
 
-    def __init__(self, mimeTypeList = ['text/html']):
+    def __init__(self, mimeTypeList = ['text/html'], compresslevel=9):
         # List of mime-types to compress
         self.mimeTypeList = mimeTypeList
+        self.compresslevel = compresslevel
 
     def beforeResponse(self):
         if not cpg.response.body:
         if (ct in self.mimeTypeList) and ('gzip' in ae):
             # Set header
             cpg.response.headerMap['Content-Encoding'] = 'gzip'
-            # Compress page
-            zbuf = StringIO.StringIO()
-            zfile = gzip.GzipFile(mode='wb', fileobj = zbuf, compresslevel = 9)
-            zfile.write(cpg.response.body)
-            zfile.close()
-            cpg.response.body = zbuf.getvalue()
+            # Return a generator that compresses the page
+            cpg.response.body = self.zip_body(cpg.response.body)
 
+    def write_gzip_header(self):
+        """
+        Adapted from the gzip.py standard module code
+        """
+        header = '\037\213'      # magic header
+        header += '\010'         # compression method
+        header += '\0'
+        header += struct.pack("<L", long(time.time()))
+        header += '\002'
+        header += '\377'
+        return header
+            
+    def write_gzip_trailer(self, crc, size):
+        footer = struct.pack("<l", crc)
+        footer += struct.pack("<L", size & 0xFFFFFFFFL)
+        return footer
 
+    def zip_body(self, body):
+        # Compress page
+        yield self.write_gzip_header()
+        crc = zlib.crc32("")
+        size = 0
+        zobj = zlib.compressobj(self.compresslevel, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0)
+        for line in body:
+            size += len(line)
+            crc = zlib.crc32(line, crc)
+            yield zobj.compress(line)
+        yield zobj.flush()
+        yield self.write_gzip_trailer(crc, size)

cherrypy/lib/filter/logdebuginfofilter.py

 import time, StringIO, pickle
 from basefilter import BaseInputFilter, BaseOutputFilter
 from cherrypy import cpg
+from itertools import chain
 
 class LogDebugInfoStartFilter(BaseInputFilter, BaseOutputFilter):
     """
     def beforeResponse(self):
         ct = cpg.response.headerMap.get('Content-Type')
         if (ct in self.mimeTypeList):
+            debuginfo = '\n'
             if self.logAsComment:
-                cpg.response.body += '<!-- '
+                debuginfo += '<!-- '
             else:
-                cpg.response.body += self.preTag
+                debuginfo += self.preTag
             logList = []
             if self.logBuildTime:
                 logList.append("Build time: %.03fs" % (
                 logList.append("Session data size: %.02fKB" % (
                     len(dumpStr)/float(1024)))
 
-            cpg.response.body += ', '.join(logList)
+            debuginfo += ', '.join(logList)
             if self.logAsComment:
-                cpg.response.body += '-->'
+                debuginfo += '-->'
 
-            if 'Content-Length' in cpg.response.headerMap:
-                cpg.response.headerMap['Content-Length'] = len(cpg.response.body)
-
+            cpg.response.body = chain(cpg.response.body, [debuginfo])

cherrypy/lib/filter/tidyfilter.py

         self.errorsToIgnore = errorsToIgnore
 
     def beforeResponse(self):
+        # the tidy filter, by its very nature it's not generator friendly, 
+        # so we just collect the body and work with it.
+        originalBody = ''.join(cpg.response.body)
+        
         ct = cpg.response.headerMap.get('Content-Type')
         if ct == 'text/html':
             pageFile = os.path.join(self.tmpDir, 'page.html')
             outFile = os.path.join(self.tmpDir, 'tidy.out')
             errFile = os.path.join(self.tmpDir, 'tidy.err')
             f = open(pageFile, 'wb')
-            f.write(cpg.response.body)
+            f.write(originalBody)
             f.close()
             encoding = cpg.response.headerMap.get('Content-Encoding', '')
             if encoding:
                     if not ignore: newErrList.append(err)
 
             if newErrList:
-                oldHtml = cpg.response.body
-                cpg.response.body = "Wrong HTML:<br>" + cgi.escape('\n'.join(newErrList)).replace('\n','<br>')
-                cpg.response.body += '<br><br>'
+                newBody = "Wrong HTML:<br>" + cgi.escape('\n'.join(newErrList)).replace('\n','<br>')
+                newBody += '<br><br>'
                 i=0
                 for line in oldHtml.splitlines():
                     i += 1
-                    cpg.response.body += "%03d - "%i + cgi.escape(line).replace('\t','    ').replace(' ','&nbsp;') + '<br>'
+                    newBody += "%03d - "%i + cgi.escape(line).replace('\t','    ').replace(' ','&nbsp;') + '<br>'
 
-                cpg.response.headerMap['Content-Length'] = len(cpg.response.body)
+                cpg.response.body = [newBody]

cherrypy/lib/filter/xmlrpcfilter.py

+##########################################################################
+## 
+## xmlrpcfilter is as open as it can be. Do whatever you like,
+## but understand that i take no responsibility for the code itself,
+## nor for wat it does and especially any derivatives and when you use
+## it, keep a copyright somewhere of me, or make notice of CherryPy
+##
+## Remco Boerma
+##
+## History:
+## 0.0.9   : 2004-12-23 made it CP2 #59 compatible (returns an iterable)
+##           Please note: as the xmlrpc doesn't know what you would want to return
+##           (and for the logic of marshalling) it will return Generator objects, as
+##           it is.. So it'll brake on that one!!
+##           NOTE: __don't try to return a Generator object to the caller__
+##           You could of course handle the generator usage internally, before sending
+##           the result. This breaks from the general cherrypy way of handling generators...
+## 0.0.8   : 2004-12-23 cpg.request.paramList should now be a filter. 
+## 0.0.7   : 2004-12-07 inserted in the experimental branch (all remco boerma till here)
+## 0.0.6   : 2004-12-02 Converted basefilter to baseinputfileter,baseoutputfilter
+## 0.0.5   : 2004-11-22 "RPC2/" now changed to "/RPC2/" with the new mapping function
+##           Gian paolo ciceri notified me with the lack of passing parameters.
+##           Thanks Gian, it's now implemented against the latest trunk.
+##           Gian also came up with the idea of lazy content-type checking: if it's sent
+##           as a header, it should be 'text/xml', if not sent at all, it should be
+##           accepted. (While this it not the xml/rpc standard, it's handy for those
+##           xml-rpc client implementations wich don't send this header)
+## 0.0.4   : 2004-11-20 in setting the path, the dot is replaces by a slash
+##           therefore the regular CP2 routines knows how to handle things, as 
+##           dots are not allowed in object names, it's varely easily adopted. 
+##           Path + method handling. The default path is 'RPC2', this one is 
+##           stripped. In case of path 'someurl' it is used for 'someurl' + method
+##           and 'someurl/someotherurl' is mapped to someurl.someotherurl + method.
+##           this way python serverproxies initialised with an url other than 
+##           just the host are handled well. I don't hope any other service would map
+##           it to 'RPC2/someurl/someotherurl', cause then it would break i think. .
+## 0.0.3   : 2004-11-19 changed some examples (includes error checking 
+##           wich returns marshalled Fault objects if the request is an RPC call.
+##           took testing code form afterRequestHeader and put it in 
+##           testValidityOfRequest to make things a little simpler. 
+##           simply log the requested function with parameters to stdout
+## 0.0.2   : 2004-11-19 the required cgi.py patch is no longer needed
+##           (thanks remi for noticing). Webbased calls to regular objects
+##           are now possible again ;) so it's no longer a dedicated xmlrpc
+##           server. The test script is also in a ready to run file named 
+##           testRPC.py along with the test server: filterExample.py
+## 0.0.1   : 2004-11-19 informing the public, dropping loads of useless
+##           tests and debugging
+## 0.0.0   : 2004-11-19 initial alpha
+## 
+##---------------------------------------------------------------------
+## 
+## EXAMPLE CODE FOR THE SERVER:
+##    from cherrypy import cpg
+##    import xmlrpcfilter
+##    class Root:
+##        _cpFilterList = [xmlrpcfilter.XmlRpcFilter(mimeTypeList = ['text/xml'])] 
+##
+##        def test(self):
+##            return `"I'm here"`
+##        test.exposed = True
+##    cpg.root = Root()
+##    cpg.server.start()
+##        
+## EXAMPLE CODE FOR THE CLIENT:
+##    import xmlrpclib
+##    server = xmlrpclib.ServerProxy('http://localhost:8080')
+##    print server.test()
+##    # results in: "I'm here"
+## 
+######################################################################
+
+from basefilter import BaseInputFilter, BaseOutputFilter
+from cherrypy import cpg
+import xmlrpclib
+
+
+class XmlRpcFilter(BaseInputFilter,BaseOutputFilter):
+    """
+    Derivative of basefilter.
+    Test to convert XMLRPC to CherryPy2 object system and reverse
+
+    PLEASE NOTE:
+
+
+    --  IN CASE TICKET #28 IS NOT RESOLVED
+        ANY XMLRPC FUNCTION NEEDS TO RETURN A PYTHON SOURCE STRING
+            use
+              return `result`
+            insted of
+              return result
+            . 
+
+    --  AS ALL REQUESTS MUST RETURN A STRING (UNTIL THE FIX IS THERE)
+        ALL METHODS ARE CALLABLE USING A REGULAR WEBBROWSER AS WELL!!
+
+    afterRequestHeader:
+        Unmarshalls the posted data to a methodname and parameters.
+            - These are stored in cpg.request.rpcMethod and cpg.request.rpcParams
+            - The method is also stored in cpg.request.path, so CP2 will find the right
+              method to call for you. Based on the root's position
+    beforeResponse:
+        Marshalls the result of the excecuted function (in cpg.response.body) to xmlrpc.
+            - Until resolved: the result must be a python souce string with the results,
+              this string is 'eval'ed to return the results. This will be resolved in the
+              future.
+            - the Content-Type and -Length are set according to the new (marshalled) data. 
+              
+
+    """
+    def testValidityOfRequest(self):
+        # test if the content-length was sent
+        result = int(cpg.request.headerMap.get('Content-Length',0)) > 0
+        result = result and cpg.request.headerMap.get('Content-Type','text/xml').lower() in ['text/xml']
+        return result
+        
+    def afterRequestHeader(self):
+        """ Called after the request header has been read/parsed"""
+##        try:
+##            x = cpg.request.isRPC # should fail!
+##            print "error: afterRequestHeader is called twice!"
+##            return 
+##        except:
+##            pass
+        cpg.request.isRPC = self.testValidityOfRequest()
+        if not cpg.request.isRPC: 
+            print 'not a valid xmlrpc call'
+            return # break this if it's not for this filter!!
+        print "xmlrpcmethod...",
+        cpg.request.parsePostData = 0
+        dataLength = int(cpg.request.headerMap.get('Content-Length',0))
+        # ought to be true:
+        # if cpg.request.method == 'POST':
+        # if not, it's probabely a webbrowser requesting the url
+        data = cpg.request.rfile.read(dataLength)
+        #else:
+        #    data = None
+
+        # for testing: an exception may be raised as well. . an xmlrpc 'Fault' would be better though.
+        try:
+            params, method = xmlrpclib.loads(data)
+        except Exception,e: 
+            params, method =  ('ERROR PARAMS',),'ERRORMETHOD'
+        cpg.request.rpcMethod, cpg.request.rpcParams = method,params
+        # patch the path. .there are only a few options:
+        # - 'RPC2' + method >> method
+        # - 'someurl' + method >> someurl.method
+        # - 'someurl/someother' + method >> someurl.someother.method
+        if not cpg.request.path.endswith('/'):
+            cpg.request.path+='/'
+        if cpg.request.path.startswith('/RPC2/'):
+            cpg.request.path=cpg.request.path[5:] ## strip the irst /rpc2
+        cpg.request.path+=str(method).replace('.','/')
+        cpg.request.paramList = list(params)
+        print "XMLRPC Filter: calling '%s' with args: '%s' " % (cpg.request.path,params)
+
+    def beforeResponse(self):
+        """ Called before starting to write response """
+        if not cpg.request.isRPC: 
+            return # it's not an RPC call, so just let it go with the normal flow
+        try:
+            print 'beforeResponse: cpg.response.body ==',`cpg.response.body` 
+            cpg.response.body = xmlrpclib.dumps((cpg.response.body[0],), methodresponse=1,allow_none=1)
+        except xmlrpclib.Fault,fault:
+            cpg.response.body = xmlrpclib.dumps(fault,allow_none=1)
+        except Exception,e:
+            print 'EXCEPTION: ',e
+        cpg.response.headerMap['Content-Type']='text/xml'
+        cpg.response.headerMap['Content-Length']=`len(cpg.response.body)`
+

cherrypy/test/testFilter1.py

 
 code = r"""
 from cherrypy import cpg
-import cherrypy.lib.filter.gzipfilter as gzipfilter
-import cherrypy.lib.filter.encodingfilter as encodingfilter
-import cherrypy.lib.filter.generatorfilter as generatorfilter
+from cherrypy.lib.filter.gzipfilter import GzipFilter
+from cherrypy.lib.filter.encodingfilter import EncodingFilter
 europoundUnicode = u'\x80\xa3'
 class Root:
     _cpFilterList = [
-        generatorfilter.GeneratorFilter(),
-        encodingfilter.EncodingFilter(),
-        gzipfilter.GzipFilter()
+        EncodingFilter(),
+        GzipFilter()
     ]
     def index(self):
         yield u"Hello,"
     print "    Testing Filters (1) ...",
     # Gzip compression doesn't always return the same exact result !
     # So we just check that the first few bytes are the same
-    helper.checkPageResult('Object mapping', infoMap, code, testList, failedList, extraRequestHeader = [("Accept-Encoding", "gzip")])
+    helper.checkPageResult('Filters', infoMap, code, testList, failedList, extraRequestHeader = [("Accept-Encoding", "gzip")])
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.