Georg Brandl avatar Georg Brandl committed 82a0a1d

Rexx lexer: add changelog entry and simplify analyse_text

Comments (0)

Files changed (3)

   * BlitzBasic (PR#197)
   * EBNF (PR#193)
   * Igor Pro (PR#172)
+  * Rexx (PR#199)
 
 - Pygments will now recognize "vim" modelines when guessing the lexer for
   a file based on content (PR#118).

pygments/lexers/other.py

 from pygments.lexer import RegexLexer, include, bygroups, using, \
      this, combined, ExtendedRegexLexer
 from pygments.token import Error, Punctuation, Literal, Token, \
-     Text, Comment, Operator, Keyword, Name, String, Number, Generic, Other, \
+     Text, Comment, Operator, Keyword, Name, String, Number, Generic, \
      Whitespace
 from pygments.util import get_bool_opt
 from pygments.lexers.web import HtmlLexer
             (r"'", String, 'string_single'),
             (r'[0-9]+(\.[0-9]+)?(e[+-]?[0-9])?', Number),
             (r'([a-z_][a-z0-9_]*)(\s*)(:)(\s*)(procedure)\b',
-             bygroups(Name.Function, Whitespace, Operator, Whitespace, Keyword.Declaration)),
+             bygroups(Name.Function, Whitespace, Operator, Whitespace,
+                      Keyword.Declaration)),
             (r'([a-z_][a-z0-9_]*)(\s*)(:)',
              bygroups(Name.Label, Whitespace, Operator)),
             include('function'),
         ]
     }
 
-    _ADDRESS_COMMAND_PATTERN = r'^\s*address\s+command\b'
-    _ADDRESS_PATTERN = r'^\s*address\s+'
-    _DO_WHILE_PATTERN = r'^\s*do\s+while\b'
-    _IF_THEN_DO_PATTERN = r'^\s*if\b.+\bthen\s+do\s*$'
-    _PROCEDURE_PATTERN = r'^\s*([a-z_][a-z0-9_]*)(\s*)(:)(\s*)(procedure)\b'
-    _ELSE_DO_PATTERN = r'\belse\s+do\s*$'
-    _PARSE_ARG_PATTERN = r'^\s*parse\s+(upper\s+)?(arg|value)\b'
-    _PATTERNS_AND_WEIGHTS = (
+    _c = lambda s: re.compile(s, re.MULTILINE)
+    _ADDRESS_COMMAND_PATTERN = _c(r'^\s*address\s+command\b')
+    _ADDRESS_PATTERN = _c(r'^\s*address\s+')
+    _DO_WHILE_PATTERN = _c(r'^\s*do\s+while\b')
+    _IF_THEN_DO_PATTERN = _c(r'^\s*if\b.+\bthen\s+do\s*$')
+    _PROCEDURE_PATTERN = _c(r'^\s*([a-z_][a-z0-9_]*)(\s*)(:)(\s*)(procedure)\b')
+    _ELSE_DO_PATTERN = _c(r'\belse\s+do\s*$')
+    _PARSE_ARG_PATTERN = _c(r'^\s*parse\s+(upper\s+)?(arg|value)\b')
+    PATTERNS_AND_WEIGHTS = (
         (_ADDRESS_COMMAND_PATTERN, 0.2),
         (_ADDRESS_PATTERN, 0.05),
         (_DO_WHILE_PATTERN, 0.1),
         (_PARSE_ARG_PATTERN, 0.2),
     )
 
-    @staticmethod
-    def _analyse_text_for_weighted_patterns(text, patternsAndWeights):
-        result = 0.0
-        lowerText = text.lower()
-        for pattern, weight in patternsAndWeights:
-            regex = re.compile(pattern, re.MULTILINE)
-            if regex.search(lowerText):
-                result += weight
-        return result
-
     def analyse_text(text):
         """
-        Check for inital comment.
+        Check for inital comment and patterns that distinguish Rexx from other
+        C-like languages.
         """
-        result = 0.0
         if re.search(r'/\*\**\s*rexx', text, re.IGNORECASE):
             # Header matches MVS Rexx requirements, this is certainly a Rexx
             # script.
-            result = 1.0
+            return 1.0
         elif text.startswith('/*'):
             # Header matches general Rexx requirements; the source code might
             # still be any language using C comments such as C++, C# or Java.
-            result = 0.01
-            result += RexxLexer._analyse_text_for_weighted_patterns(
-                text, RexxLexer._PATTERNS_AND_WEIGHTS)
-            result = min(result, 1.0)
-        assert 0.0 <= result <= 1.0
-        return result
+            lowerText = text.lower()
+            result = sum(weight
+                         for (pattern, weight) in RexxLexer.PATTERNS_AND_WEIGHTS
+                         if pattern.search(lowerText)) + 0.01
+            return min(result, 1.0)

tests/test_lexers_other.py

     :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
+
 import glob
 import os
 import unittest
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.