Commits

Olemis Lang committed 2ba1b6f

Added MIME converter to load GViz DataTable(s) from CSV and TSV. Tested ?

Comments (0)

Files changed (5)

trac-dev/gviz/TODO

 
 X Render GViz provider list on accessing `/gviz` path.
 
-+ Include helper to convert CSV to GViz API JSON format.
+X Implement CSV and TSV encoders by using `csv` module.
 
-+ Implement CSV and TSV encoders by using `csv` module.
++ Include helper to convert CSV attachments to GViz API JSON format.
 
 + Integrate `IGVizTableEncoder` with MIME view.
 
 
 + Provide `dutest` as an external dependency.
 
+- Include helper to convert CSV files under version control to GViz 
+  API JSON format.
+
+- Write further tests for `tracgviz.xsv.GvizXSVConverter`.
+
 - Add support for `outFileName` option in TSV encoder.
 
 - Allow encoders to introduce headers in the HTTP response.
 
 - Migrate `fmtvalue` helper function from encoders towards `tracgviz.util`.
 
+- Try to use values instead of formatted values in CSV and TSV encoders.
+
 - Include helper to convert OOo files to GViz API JSON format.
 
 - Don't skip silently the exceptions raised while executing 
   expose data under version control (`svn` + `hg` + `git` ? + `mtn` + 
   `bzr` ? ) like `VersionControlRPC`. Include example repos.
 
+- Unify test loaders.
+
+- Implement clonation of `dutest.DocTestLoader`s.
+
+- Select modules to be tested.
+
 - Implement TracAnalytics.
 
 - Add data source and infrastructure to convert files containing 
 
 - Fix the issue with selecting changed tickets since timestamp.
 
-- Ensure that columns are always arranged in the same order (i.e. use 
-  ordered dicts).
-
 - Optimize the calls to TracRPC's Method objects by preventing 
   them from building lists out of generators.
 
 
 - Execute reports defined using URLs pointing to saved custom queries.
 
-- Implement architure in order to export data.
+- Implement architecture in order to export data.
 
 - Export data to Google Spreadsheets.
 

trac-dev/gviz/tracgviz/__init__.py

     from search import *
     from timeline import *
     from vcs import *
+    from xsv import GvizXSVConverter
 #    from ig import *
     msg = 'Ok'
 except Exception, exc:

trac-dev/gviz/tracgviz/testing/__init__.py

 from util import dummy_request
 
 # Hide this module from tracebacks written into test results.
-# __unittest = True
+__unittest = True
 
 class DocTestTracLoader(DocTestLoader):
   r"""A generic XUnit loader that allows to load doctests written 

trac-dev/gviz/tracgviz/testing/test_rpc.py

 #------------------------------------------------------
 
 def test_suite():
+  from doctest import NORMALIZE_WHITESPACE, ELLIPSIS, REPORT_UDIFF
   from dutest import MultiTestLoader
-  from doctest import NORMALIZE_WHITESPACE, ELLIPSIS, REPORT_UDIFF
   from unittest import defaultTestLoader
   
   from __init__ import DocTestRpcLoader, ticket_data

trac-dev/gviz/tracgviz/xsv.py

 __author__ = 'Olemis Lang'
 
 from trac.core import Component, implements
+from trac.mimeview.api import IContentConverter
+from trac.util.translation import _
 
-from csv import writer, QUOTE_MINIMAL
+from csv import writer, QUOTE_MINIMAL, reader
 from itertools import izip
 
 from api import IGVizTableEncoder
 from util import render_gviz_value, iter_table_data, StringIO
 
+__all__ = 'GVizCSVEncoder', 'GVizTSVEncoder', 'GvizXSVConverter'
+
+#--------------------------------------------------
+#   GViz Encoders
+#--------------------------------------------------
+
 class GVizSeparatedValuesEncoder(Component):
     r"""Generic encoder for formats where values are separated by a 
     fixed value, and rows are lines.
         """
         return 'text/tab-separated-values;charset=utf-16'
 
+#--------------------------------------------------
+#   MIME conversions
+#--------------------------------------------------
+
+class GvizXSVConverter(Component):
+    r"""Load CSV and TSV data into GViz data tables. 
+    You can convert data using Trac MIME API like this (CSV input).
+    
+    >>> content = r'''Task Name,Due time
+    ... milestone1,2009-05-02 13:15:30
+    ... milestone2,
+    ... milestone3,
+    ... milestone4,
+    ... here,2009-08-23 05:00:00
+    ... '''
+    >>> from trac.mimeview.api import Mimeview
+    >>> (cols, data), mimetype, ext = Mimeview(env).convert_content(
+    ...                                             req, 'text/csv', \
+    ...                                             content, 'trac.gviz')
+    >>> mimetype, ext 
+    ('trac.gviz', 'gviz')
+    >>> cols
+    [('Task Name', 'string'), ('Due time', 'string')]
+    >>> data
+    <_csv.reader object at ...>
+    >>> for row in data :
+    ...   for x in row :
+    ...     print repr(x),
+    ...   print
+    ...
+    'milestone1' '2009-05-02 13:15:30'
+    'milestone2' ''
+    'milestone3' ''
+    'milestone4' ''
+    'here' '2009-08-23 05:00:00'
+    """
+    implements(IContentConverter)
+    
+    # IContentConverter methods
+    def get_supported_conversions(self):
+        yield ('csv2gviz', _('GViz Data Table (Comma-delimited Text)'), 
+                'gviz', 'text/csv', 'trac.gviz', 8)
+        yield ('tab2gviz', _('GViz Data Table (Tab-delimited Text)'), 
+                'gviz', 'text/tab-separated-values', 'trac.gviz', 8)
+    
+    def convert_content(self, req, mimetype, content, key):
+        if key == 'csv2gviz':
+            sep = ','
+        elif key == 'tab2gviz':
+            sep = '\t'
+        return self.load_data(req, content, sep)
+
+    def load_data(self, req, data, sep=','):
+        r"""Prepare data delimited by a special character to be loaded 
+        into instances of `DataTable`. It is possible to customize the 
+        transformation by specifying the parameters 
+        `csv_doublequote`, `csv_escapechar`, `csv_lineterminator`, 
+        `csv_quotechar`, `csv_quoting`, `csv_skipinitialspace` in the 
+        input request. They are used to prepare the input formatting 
+        parameters as defined by `csv` module. Notice that the 
+        delimiter is ignored.
+        
+        @return       a binary tuple of the form (`schema`, `data`) 
+                      where `schema` is built considering the values 
+                      in the first input row and `data` is an iterator 
+                      returning the rest of the values.
+        """
+        content = StringIO(data)
+        csv_args = dict([x[4:], v] for x, v in req.args.iteritems() \
+                                    if x.startswith('csv_'))
+        csv_args['delimiter'] = sep
+        r = reader(content, **csv_args)
+        try :
+          columns = [(x, 'string') for x in r.next()]
+        except StopIteration :
+          return None
+        else :
+          return (columns, r), 'trac.gviz'
+
+#------------------------------------------------------
+#   Global Testing
+#------------------------------------------------------
+
+def test_suite():
+  from doctest import ELLIPSIS, NORMALIZE_WHITESPACE, REPORT_UDIFF
+  from testing.dutest import MultiTestLoader
+  from unittest import defaultTestLoader
+  from string import whitespace
+  import sys
+  
+  from testing import DocTestTracLoader
+  
+  l = MultiTestLoader([defaultTestLoader, \
+                        DocTestTracLoader(
+                            enable=['trac.[a-uw-z]*', 'tracrpc.*', \
+                                        'tracgviz.*'], \
+                            default_data=False,
+                            optionflags=ELLIPSIS \
+                                        | NORMALIZE_WHITESPACE \
+                                        | REPORT_UDIFF \
+                          )])
+  return l.loadTestsFromModule(sys.modules[__name__])
+