Commits

Olemis Lang committed 5c3b932

GViz QL : Improved unicode handling bby relying upon utf-8 conversions . Fixes #121

Comments (0)

Files changed (1)

trac-dev/gviz/tracgviz/gvizql.py

 from tracgviz.util import groupby
 from tracgviz.util.parsing import OperatorPrecedenceParser,  \
     InvalidParserConfiguration
+from trac.util.text import to_unicode
 
 #------------------------------------------------------
 #   GVizQL clause handlers 
     by column name.
     """
     if schema is None:
+      # FIXME: str(colnm) => colnm.encode('utf-8')
       return lambda colnm: (str(colnm), None)
 
     schcols = dict([col[0], i] for i, col in enumerate(schema))
     elif not isinstance(seq, Sequence):
       seq = Sequence([seq])
     self.cols_mapping = seq
+    # FIXME: str(colnm) => colnm.encode('utf-8')
     self.cols = [self._resolve_schema(c, lambda colnm: (str(colnm), None))[0] 
         for c in self.cols_mapping ] if seq is not None else None
     # Activate select clause. May be turned off e.g. by PIVOT clause
 
     FROM clause is deprecated and has been removed from the language.
     """
+    # FIXME: str(colnm) => colnm.encode('utf-8')
     self.basetable = name['schema'](lambda colnm: (str(colnm), None))[0]
 
   def transform(self, schema, data):
           (" Got %s" % (expr['schema'][1])) if not expr['is_schema_callable']
               else '')
     self.expr = expr
+    # FIXME: str(colnm) => colnm.encode('utf-8')
     self.filter = self._resolve_schema(expr, lambda colnm: (str(colnm), None))[0]
 
   def transform(self, schema, data):
     if not isinstance(seq, Sequence):
       seq = Sequence([seq])
     self.groups_def = seq
+    # FIXME: str(colnm) => colnm.encode('utf-8')
     self.aggregate = [self._resolve_schema(c, lambda colnm: (str(colnm), None))[0] 
         for c in self.groups_def ]
 
     if not isinstance(seq, Sequence):
       seq = Sequence([seq])
     self.pivot_def = seq
+    # FIXME: str(colnm) => colnm.encode('utf-8')
     self.pivot = [self._resolve_schema(c, lambda colnm: (str(colnm), None))[0] 
         for c in self.pivot_def ]
     self.select_handler = None
     def pivot_label(value):
         """Generate label for group key in pivot clause
         """
-        return ','.join(x.encode('utf-8') if x is not None else 'null' for x in value)
+        return ','.join(unicode(x).encode('utf-8') if x is not None else 'null'
+                        for x in value)
 
     if hasattr(schema, '__schema__'):
         gb_schema = schema
       seq = Sequence([(seq, True)])
     self.order_def = [c if isinstance(c, tuple) else (c, True) for c in seq]
     self.sort = [
-        (self._resolve_schema(c, lambda colnm: (str(colnm), None))[0], is_asc) 
-        for c, is_asc in self.order_def ]
+        (self._resolve_schema(c, lambda colnm: (colnm.encode('utf-8'), None))[0],
+         is_asc) for c, is_asc in self.order_def ]
 
   def transform(self, schema, data):
     r"""Sort rows in base result set by evaluating target expressions.
     r"""Initialize this clause with a list of option names.
     """
     if not isinstance(seq, Sequence):
-      get_col_schema = lambda _colnm: (str(_colnm), None)
+      get_col_schema = lambda _colnm: (_colnm.encode('utf-8'), None)
       seq = seq['schema'](get_col_schema)[0]
       seq = Sequence([seq])
     self.opts = seq
   def handle_str(self, value):
     return dict(
         eval=literal_eval(value[1]), 
-        schema=(str(value[1]), 'string'),
+        schema=(value[1].encode('utf-8'), 'string'),
         is_eval_callable=False,
         is_schema_callable=False)
 
   def handle_bfunc(self, funcnm, _, colnm, __):
     funcnm = str(funcnm[1])
     colnm = colnm[1]
+    # FIXME: str(colnm) => colnm.encode('utf-8')
     col_schema = colnm['schema'](lambda colnm: (str(colnm), None) )
     try:
       f = self.agg_funcs['agg_' + funcnm]
           return f(row.__group__, get_group_value, col_schema[0])
         except GVizUnknownColumn:
           # Target column is not available
-          raise GVizUnknownColumn('%s(%s) : No such column in GROUP BY, PIVOT'% \
+          raise GVizUnknownColumn('%s(%s) : No such column in GROUP BY, PIVOT'%\
               (funcnm, col_schema[0]) )
         except GVizRuntimeError, exc:
-          raise GVizRuntimeError(func_label + ' : ' + str(exc))
+          msg = to_unicode(exc).encode('utf-8')
+          raise GVizRuntimeError(func_label + ' : ' + msg)
+                                 
 
     if not return_type:
       def agg_schema(get_col_schema):
           raise GVizRuntimeError('%s(%s) : No such column in GROUP BY, PIVOT'% \
               (funcnm, col_schema[0]) )
         except GVizRuntimeError, exc:
-          raise GVizRuntimeError(func_label + ' : ' + str(exc))
+          msg = to_unicode(exc).encode('utf-8')
+          raise GVizRuntimeError(func_label + ' : ' + msg)
 
     return dict(
         eval=eval_aggregate,
     is_eval_callable, eval_func = self._eval_expr(f, [lexpr, rexpr])
     return_type = 'number' if op in '-+*/' else 'boolean'
     if lexpr['is_schema_callable']:
+      # FIXME: str(colnm) => colnm.encode('utf-8')
       lschema = lexpr['schema'](lambda colnm : (str(colnm), None))
     else:
       lschema = lexpr['schema']
     if rexpr['is_schema_callable']:
+      # FIXME: str(colnm) => colnm.encode('utf-8')
       rschema = rexpr['schema'](lambda colnm : (str(colnm), None))
     else:
       rschema = rexpr['schema']
     colnm = colnm[1]
     if colnm.startswith('`'):
       colnm = colnm[1:-1]
-    return (str(colnm), literal_eval(label[1]))
+    return (colnm.encode('utf-8'), literal_eval(label[1]))
 
   def handle_lblseq(self, labels, _, label_expr):
     labels = labels[1]
     return labels
 
   def handle_column(self, colnm):
-    return colnm[1]['schema'](lambda _colnm: (str(_colnm), None))[0]
+    return colnm[1]['schema'](lambda _colnm: (_colnm.encode('utf-8'), None))[0]
 
   def handle_colseq(self, colseq, _, colnm):
     colseq = colseq[1]
     colnm = colnm[1]
-    get_col_schema = lambda _colnm: (str(_colnm), None)
+    get_col_schema = lambda _colnm: (_colnm.encode('utf-8'), None)
     if isinstance(colseq, self.Sequence):
       colnm = colnm['schema'](get_col_schema)[0]
       colseq.append(colnm)
   # Internal methods
 
   def _function_label(self, funcnm, seq):
+    # FIXME: str(colnm) => colnm.encode('utf-8')
     return '%s(%s)' % (funcnm, 
         ', '.join(x['schema'][0] if not x['is_schema_callable'] \
             else x['schema'](lambda colnm: (str(colnm), None))[0] \