1. Jason R. Coombs
  2. ply

Commits

Kristoffer Ellersgaard Koch  committed a2bf37c

When using @TOKEN decorator, use another attribute than __doc__ to store regex

  • Participants
  • Parent commits 719d417
  • Branches default

Comments (0)

Files changed (1)

File ply/lex.py

View file
                     self.error = 1
                     continue
 
-                if not f.__doc__:
+                if not _get_regex(f):
                     self.log.error("%s:%d: No regular expression defined for rule '%s'",file,line,f.__name__)
                     self.error = 1
                     continue
 
                 try:
-                    c = re.compile("(?P<%s>%s)" % (fname,f.__doc__), re.VERBOSE | self.reflags)
+                    c = re.compile("(?P<%s>%s)" % (fname,_get_regex(f)), re.VERBOSE | self.reflags)
                     if c.match(""):
                         self.log.error("%s:%d: Regular expression for rule '%s' matches empty string", file,line,f.__name__)
                         self.error = 1
                 except re.error:
                     _etype, e, _etrace = sys.exc_info()
                     self.log.error("%s:%d: Invalid regular expression for rule '%s'. %s", file,line,f.__name__,e)
-                    if '#' in f.__doc__:
+                    if '#' in _get_regex(f):
                         self.log.error("%s:%d. Make sure '#' in rule '%s' is escaped with '\\#'",file,line, f.__name__)
                     self.error = 1
 
         for fname, f in linfo.funcsym[state]:
             line = func_code(f).co_firstlineno
             file = func_code(f).co_filename
-            regex_list.append("(?P<%s>%s)" % (fname,f.__doc__))
+            regex_list.append("(?P<%s>%s)" % (fname,_get_regex(f)))
             if debug:
-                debuglog.info("lex: Adding rule %s -> '%s' (state '%s')",fname,f.__doc__, state)
+                debuglog.info("lex: Adding rule %s -> '%s' (state '%s')",fname,_get_regex(f), state)
 
         # Now add all of the simple rules
         for name,r in linfo.strsym[state]:
         if not tok: break
         sys.stdout.write("(%s,%r,%d,%d)\n" % (tok.type, tok.value, tok.lineno,tok.lexpos))
 
+def _get_regex(func):
+    if hasattr(func, 'regex'):
+        return func.regex
+    # Fall back to docstring if @TOKEN has not been used
+    return func.__doc__
+
 # -----------------------------------------------------------------------------
 # @TOKEN(regex)
 #
 def TOKEN(r):
     def set_doc(f):
         if hasattr(r,"__call__"):
-            f.__doc__ = r.__doc__
+            f.regex = _get_regex(r)
         else:
-            f.__doc__ = r
+            f.regex = r
         return f
     return set_doc