Mike Orr avatar Mike Orr committed 8228561

Upgrade to Markdown 1.7. Warn about the need to escape untrusted HTML.

Comments (0)

Files changed (4)

   groups.  select() now uses this automatically.
 * webhelpers.converters:
   - New helper transpose().
+* webhelpers.markdown:
+  - Upgrade to Markdown 1.7.
+  - Add a warning about escaping untrusted HTML to 
+    webhelpers.html.converters.markdown() docstring.
+  - Did not include Markdown's extensions due to circular import issues.
+    Use the full Markdown package if you want footnotes or RSS.
 * webhelpers.media:
   - New module for muiltimedia helpers.  Initial functions determine the size
     of an image and choose a scaling factor.

tests/test_converters.py

 
 [Michel Fortin][] has ported to Markdown to PHP.
     """
-    eq_('\n\n<h2>Introduction</h2>\n<p>Markdown is a text-to-HTML conversion tool for web writers.\n</p>\n\n<h2>Acknowledgements <a id="acknowledgements" /></h2>\n<p>[Michel Fortin][] has ported to Markdown to PHP.\n</p>\n\n\n',
-                markdown(markdown_text))
+    eq_(markdown(markdown_text), literal(u'<h2>Introduction</h2>\n<p>Markdown is a text-to-HTML conversion tool for web writers.\n</p>\n\n<h2>Acknowledgements <a id="acknowledgements" /></h2>\n<p>[Michel Fortin][] has ported to Markdown to PHP.\n</p>'))
+
+def test_markdown_embedded_html():
+    markdown_text = 'This text includes <script>Javascript</script> & stuff.'
+    expected = literal(u'<p>This text includes &lt;script&gt;Javascript&lt;/script&gt; &amp; stuff.\n</p>')
+    eq_(markdown(markdown_text, safe_mode="escape"), expected)
+
 
 def test_nl2br():
     eq_(u'A B<br />\nC D<br />\n<br />\nE F', nl2br("A B\nC D\r\n\r\nE F"))

webhelpers/html/converters.py

 """
 import re
 
-from webhelpers.html import HTML, literal, lit_sub
+from webhelpers.html import HTML, escape, literal, lit_sub
 import webhelpers.textile as textile
 import webhelpers.markdown as _markdown
 
     This function uses the `Python MarkDown library 
     <http://www.freewisdom.org/projects/python-markdown/>`_
     which is included with WebHelpers.
-    
+
+    IMPORTANT:
+    If your source text is untrusted and may contain malicious HTML markup,
+    pass ``safe_mode="escape"`` to escape it, ``safe_mode="replace"`` to
+    replace it with a scolding message, or ``safe_mode="remove"`` to strip it.
     """
     return literal(_markdown.markdown(text, **kwargs))
 

webhelpers/markdown.py

 #!/usr/bin/env python
 
+version = "1.7"
+version_info = (1,7,0,"rc-2")
+__revision__ = "$Rev: 72 $"
+
+"""
+Python-Markdown
+===============
+
+Converts Markdown to HTML.  Basic usage as a module:
+
+    import markdown
+    md = Markdown()
+    html = md.convert(your_text_string)
+
+See http://www.freewisdom.org/projects/python-markdown/ for more
+information and instructions on how to extend the functionality of the
+script.  (You might want to read that before you try modifying this
+file.)
+
+Started by [Manfred Stienstra](http://www.dwerg.net/).  Continued and
+maintained  by [Yuri Takhteyev](http://www.freewisdom.org) and [Waylan
+Limberg](http://achinghead.com/).
+
+Contact: yuri [at] freewisdom.org
+         waylan [at] gmail.com
+
+License: GPL 2 (http://www.gnu.org/copyleft/gpl.html) or BSD
+
+"""
+
+
+import re, sys, codecs
+
+from logging import getLogger, StreamHandler, Formatter, \
+                    DEBUG, INFO, WARN, ERROR, CRITICAL
+
+
+MESSAGE_THRESHOLD = CRITICAL
+
+
+# Configure debug message logger (the hard way - to support python 2.3)
+logger = getLogger('MARKDOWN')
+logger.setLevel(DEBUG) # This is restricted by handlers later
+console_hndlr = StreamHandler()
+formatter = Formatter('%(name)s-%(levelname)s: "%(message)s"')
+console_hndlr.setFormatter(formatter)
+console_hndlr.setLevel(MESSAGE_THRESHOLD)
+logger.addHandler(console_hndlr)
+
+
+def message(level, text):
+    ''' A wrapper method for logging debug messages. '''
+    logger.log(level, text)
+
+
+# --------------- CONSTANTS YOU MIGHT WANT TO MODIFY -----------------
+
+TAB_LENGTH = 4            # expand tabs to this many spaces
+ENABLE_ATTRIBUTES = True  # @id = xyz -> <... id="xyz">
+SMART_EMPHASIS = 1        # this_or_that does not become this<i>or</i>that
+HTML_REMOVED_TEXT = "[HTML_REMOVED]" # text used instead of HTML in safe mode
+
+RTL_BIDI_RANGES = ( (u'\u0590', u'\u07FF'),
+                    # from Hebrew to Nko (includes Arabic, Syriac and Thaana)
+                    (u'\u2D30', u'\u2D7F'),
+                    # Tifinagh
+                    )
+
+# Unicode Reference Table:
+# 0590-05FF - Hebrew
+# 0600-06FF - Arabic
+# 0700-074F - Syriac
+# 0750-077F - Arabic Supplement
+# 0780-07BF - Thaana
+# 07C0-07FF - Nko
+
+BOMS = { 'utf-8': (codecs.BOM_UTF8, ),
+         'utf-16': (codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE),
+         #'utf-32': (codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE)
+         }
+
+def removeBOM(text, encoding):
+    convert = isinstance(text, unicode)
+    for bom in BOMS[encoding]:
+        bom = convert and bom.decode(encoding) or bom
+        if text.startswith(bom):
+            return text.lstrip(bom)
+    return text
+
 # The following constant specifies the name used in the usage
 # statement displayed for python versions lower than 2.3.  (With
 # python2.3 and higher the usage statement is generated by optparse
 # and uses the actual name of the executable called.)
 
 EXECUTABLE_NAME_FOR_USAGE = "python markdown.py"
-
-SPEED_TEST = 0
-
-"""
-====================================================================
-IF YOA ARE LOOKING TO EXTEND MARKDOWN, SEE THE "FOOTNOTES" SECTION
-====================================================================
-
-Python-Markdown
-===============
-
-Converts Markdown to HTML.  Basic usage as a module:
-
-    import markdown
-    html = markdown.markdown(your_text_string)
-
-Started by [Manfred Stienstra](http://www.dwerg.net/).  Continued and
-maintained  by [Yuri Takhteyev](http://www.freewisdom.org).
-
-Project website: http://www.freewisdom.org/projects/python-markdown
-Contact: yuri [at] freewisdom.org
-
-License: GPL 2 (http://www.gnu.org/copyleft/gpl.html) or BSD
-
-Version: 1.6a (October 12, 2006)
-
-
-For changelog, see end of file
-"""
-
-import re, sys, os, random, codecs
-
-# set debug level: 3 none, 2 critical, 1 informative, 0 all
-(VERBOSE, INFO, CRITICAL, NONE) = range(4)
-
-MESSAGE_THRESHOLD = CRITICAL
-
-def message(level, text) :
-    if level >= MESSAGE_THRESHOLD :
-        print text
-
-
-# --------------- CONSTANTS YOU MIGHT WANT TO MODIFY -----------------
-
-# all tabs will be expanded to up to this many spaces
-TAB_LENGTH = 4
-ENABLE_ATTRIBUTES = 1
-SMART_EMPHASIS = 1
+                    
 
 # --------------- CONSTANTS YOU _SHOULD NOT_ HAVE TO CHANGE ----------
 
                         'form', 'fieldset', 'iframe', 'math', 'ins',
                         'del', 'hr', 'hr/', 'style']
 
-def is_block_level (tag) :
+def isBlockLevel (tag):
     return ( (tag in BLOCK_LEVEL_ELEMENTS) or
              (tag[0] == 'h' and tag[1] in "0123456789") )
 
 want. It also adds extra white space when converting DOM to string
 """
 
+ENTITY_NORMALIZATION_EXPRESSIONS = [ (re.compile("&"), "&amp;"),
+                                     (re.compile("<"), "&lt;"),
+                                     (re.compile(">"), "&gt;")]
 
-class Document :
+ENTITY_NORMALIZATION_EXPRESSIONS_SOFT = [ (re.compile("&(?!\#)"), "&amp;"),
+                                     (re.compile("<"), "&lt;"),
+                                     (re.compile(">"), "&gt;"),
+                                     (re.compile("\""), "&quot;")]
 
-    def appendChild(self, child) :
+
+def getBidiType(text):
+
+    if not text: return None
+
+    ch = text[0]
+
+    if not isinstance(ch, unicode) or not ch.isalpha():
+        return None
+
+    else:
+
+        for min, max in RTL_BIDI_RANGES:
+            if ( ch >= min and ch <= max ):
+                return "rtl"
+        else:
+            return "ltr"
+
+
+class Document:
+
+    def __init__ (self):
+        self.bidi = "ltr"
+
+    def appendChild(self, child):
         self.documentElement = child
+        child.isDocumentElement = True
         child.parent = self
         self.entities = {}
 
-    def createElement(self, tag, textNode=None) :
+    def setBidi(self, bidi):
+        if bidi:
+            self.bidi = bidi
+
+    def createElement(self, tag, textNode=None):
         el = Element(tag)
         el.doc = self
-        if textNode :
+        if textNode:
             el.appendChild(self.createTextNode(textNode))
         return el
 
-    def createTextNode(self, text) :
+    def createTextNode(self, text):
         node = TextNode(text)
         node.doc = self
         return node
             self.entities[entity] = EntityReference(entity)
         return self.entities[entity]
 
-    def toxml (self) :
+    def createCDATA(self, text):
+        node = CDATA(text)
+        node.doc = self
+        return node
+
+    def toxml (self):
         return self.documentElement.toxml()
 
-    def normalizeEntities(self, text) :
+    def normalizeEntities(self, text, avoidDoubleNormalizing=False):
 
-        pairs = [ ("&", "&amp;"),
-                  ("<", "&lt;"),
-                  (">", "&gt;"),
-                  ("\"", "&quot;")]
+        if avoidDoubleNormalizing:
+            regexps = ENTITY_NORMALIZATION_EXPRESSIONS_SOFT
+        else:
+            regexps = ENTITY_NORMALIZATION_EXPRESSIONS
 
-
-        for old, new in pairs :
-            text = text.replace(old, new)
+        for regexp, substitution in regexps:
+            text = regexp.sub(substitution, text)
         return text
 
-    def find(self, test) :
+    def find(self, test):
         return self.documentElement.find(test)
 
-    def unlink(self) :
+    def unlink(self):
         self.documentElement.unlink()
         self.documentElement = None
 
 
-class Element :
+class CDATA:
+
+    type = "cdata"
+
+    def __init__ (self, text):
+        self.text = text
+
+    def handleAttributes(self):
+        pass
+
+    def toxml (self):
+        return "<![CDATA[" + self.text + "]]>"
+
+class Element:
 
     type = "element"
 
-    def __init__ (self, tag) :
+    def __init__ (self, tag):
 
         self.nodeName = tag
         self.attributes = []
         self.attribute_values = {}
         self.childNodes = []
+        self.bidi = None
+        self.isDocumentElement = False
 
-    def unlink(self) :
-        for child in self.childNodes :
-            if child.type == "element" :
+    def setBidi(self, bidi):
+
+        if bidi:
+
+            orig_bidi = self.bidi
+
+            if not self.bidi or self.isDocumentElement:
+                # Once the bidi is set don't change it (except for doc element)
+                self.bidi = bidi
+                self.parent.setBidi(bidi)
+
+
+    def unlink(self):
+        for child in self.childNodes:
+            if child.type == "element":
                 child.unlink()
         self.childNodes = None
 
-    def setAttribute(self, attr, value) :
-        if not attr in self.attributes :
+    def setAttribute(self, attr, value):
+        if not attr in self.attributes:
             self.attributes.append(attr)
 
         self.attribute_values[attr] = value
 
-    def insertChild(self, position, child) :
+    def insertChild(self, position, child):
         self.childNodes.insert(position, child)
         child.parent = self
 
-    def removeChild(self, child) :
+    def removeChild(self, child):
         self.childNodes.remove(child)
 
-    def replaceChild(self, oldChild, newChild) :
+    def replaceChild(self, oldChild, newChild):
         position = self.childNodes.index(oldChild)
         self.removeChild(oldChild)
         self.insertChild(position, newChild)
 
-    def appendChild(self, child) :
+    def appendChild(self, child):
         self.childNodes.append(child)
         child.parent = self
 
-    def handleAttributes(self) :
+    def handleAttributes(self):
         pass
 
-    def find(self, test, depth=0) :
+    def find(self, test, depth=0):
         """ Returns a list of descendants that pass the test function """
         matched_nodes = []
-        for child in self.childNodes :
-            if test(child) :
+        for child in self.childNodes:
+            if test(child):
                 matched_nodes.append(child)
-            if child.type == "element" :
+            if child.type == "element":
                 matched_nodes += child.find(test, depth+1)
         return matched_nodes
 
     def toxml(self):
-        if ENABLE_ATTRIBUTES :
+        if ENABLE_ATTRIBUTES:
             for child in self.childNodes:
                 child.handleAttributes()
+
         buffer = ""
-        if self.nodeName in ['h1', 'h2', 'h3', 'h4'] :
+        if self.nodeName in ['h1', 'h2', 'h3', 'h4']:
             buffer += "\n"
-        elif self.nodeName in ['li'] :
+        elif self.nodeName in ['li']:
             buffer += "\n "
+
+        # Process children FIRST, then do the attributes
+
+        childBuffer = ""
+
+        if self.childNodes or self.nodeName in ['blockquote']:
+            childBuffer += ">"
+            for child in self.childNodes:
+                childBuffer += child.toxml()
+            if self.nodeName == 'p':
+                childBuffer += "\n"
+            elif self.nodeName == 'li':
+                childBuffer += "\n "
+            childBuffer += "</%s>" % self.nodeName
+        else:
+            childBuffer += "/>"
+
+
+            
         buffer += "<" + self.nodeName
-        for attr in self.attributes :
+
+        if self.nodeName in ['p', 'li', 'ul', 'ol',
+                             'h1', 'h2', 'h3', 'h4', 'h5', 'h6']:
+
+            if not self.attribute_values.has_key("dir"):
+                if self.bidi:
+                    bidi = self.bidi
+                else:
+                    bidi = self.doc.bidi
+                    
+                if bidi=="rtl":
+                    self.setAttribute("dir", "rtl")
+        
+        for attr in self.attributes:
             value = self.attribute_values[attr]
-            value = self.doc.normalizeEntities(value)
+            value = self.doc.normalizeEntities(value,
+                                               avoidDoubleNormalizing=True)
             buffer += ' %s="%s"' % (attr, value)
-        if self.childNodes or self.nodeName in ['blockquote']:
-            buffer += ">"
-            for child in self.childNodes :
-                buffer += child.toxml()
-            if self.nodeName == 'p' :
-                buffer += "\n"
-            elif self.nodeName == 'li' :
-                buffer += "\n "
-            buffer += "</%s>" % self.nodeName
-        else :
-            buffer += "/>"
-        if self.nodeName in ['p', 'li', 'ul', 'ol',
+
+
+        # Now let's actually append the children
+
+        buffer += childBuffer
+
+        if self.nodeName in ['p', 'br ', 'li', 'ul', 'ol',
                              'h1', 'h2', 'h3', 'h4'] :
             buffer += "\n"
 
         return buffer
 
 
-class TextNode :
+class TextNode:
 
     type = "text"
     attrRegExp = re.compile(r'\{@([^\}]*)=([^\}]*)}') # {@id=123}
 
-    def __init__ (self, text) :
+    def __init__ (self, text):
         self.value = text        
 
-    def attributeCallback(self, match) :
+    def attributeCallback(self, match):
+
         self.parent.setAttribute(match.group(1), match.group(2))
 
-    def handleAttributes(self) :
+    def handleAttributes(self):
         self.value = self.attrRegExp.sub(self.attributeCallback, self.value)
 
-    def toxml(self) :
+    def toxml(self):
+
         text = self.value
+
+        self.parent.setBidi(getBidiType(text))
+        
         if not text.startswith(HTML_PLACEHOLDER_PREFIX):
-            if self.parent.nodeName == "p" :
+            if self.parent.nodeName == "p":
                 text = text.replace("\n", "\n   ")
             elif (self.parent.nodeName == "li"
                   and self.parent.childNodes[0]==self):
 Preprocessors munge source text before we start doing anything too
 complicated.
 
-Each preprocessor implements a "run" method that takes a pointer to a list of lines of the document,
-modifies it as necessary and returns either the same pointer or a
-pointer to a new list.  Preprocessors must extend
-markdown.Preprocessor.
+There are two types of preprocessors: TextPreprocessor and Preprocessor.
 
 """
 
 
-class Preprocessor :
-    pass
+class TextPreprocessor:
+    '''
+    TextPreprocessors are run before the text is broken into lines.
+    
+    Each TextPreprocessor implements a "run" method that takes a pointer to a
+    text string of the document, modifies it as necessary and returns
+    either the same pointer or a pointer to a new string.  
+    
+    TextPreprocessors must extend markdown.TextPreprocessor.
+    '''
 
+    def run(self, text):
+        pass
 
-class HeaderPreprocessor (Preprocessor):
 
-    """
-       Replaces underlined headers with hashed headers to avoid
-       the nead for lookahead later.
-    """
+class Preprocessor:
+    '''
+    Preprocessors are run after the text is broken into lines.
 
-    def run (self, lines) :
+    Each preprocessor implements a "run" method that takes a pointer to a
+    list of lines of the document, modifies it as necessary and returns
+    either the same pointer or a pointer to a new list.  
+    
+    Preprocessors must extend markdown.Preprocessor.
+    '''
 
-        i = -1
-        while i+1 < len(lines) :
-            i = i+1
-            if not lines[i].strip() :
-                continue
+    def run(self, lines):
+        pass
+ 
 
-            if lines[i].startswith("#") :
-                lines.insert(i+1, "\n")
-
-            if (i+1 <= len(lines)
-                  and lines[i+1]
-                  and lines[i+1][0] in ['-', '=']) :
-
-                underline = lines[i+1].strip()
-
-                if underline == "="*len(underline) :
-                    lines[i] = "# " + lines[i].strip()
-                    lines[i+1] = ""
-                elif underline == "-"*len(underline) :
-                    lines[i] = "## " + lines[i].strip()
-                    lines[i+1] = ""
-
-        #for l in lines :
-        #    print l.encode('utf8')
-        #sys.exit(0)
-
-        return lines
-
-HEADER_PREPROCESSOR = HeaderPreprocessor()
-
-class LinePreprocessor (Preprocessor):
-    """Deals with HR lines (needs to be done before processing lists)"""
-
-    def run (self, lines) :
-        for i in range(len(lines)) :
-            if self._isLine(lines[i]) :
-                lines[i] = "<hr />"
-        return lines
-
-    def _isLine(self, block) :
-        """Determines if a block should be replaced with an <HR>"""
-        if block.startswith("    ") : return 0  # a code block
-        text = "".join([x for x in block if not x.isspace()])
-        if len(text) <= 2 :
-            return 0
-        for pattern in ['isline1', 'isline2', 'isline3'] :
-            m = RE.regExp[pattern].match(text)
-            if (m and m.group(1)) :
-                return 1
-        else:
-            return 0
-
-LINE_PREPROCESSOR = LinePreprocessor()
-
-
-class LineBreaksPreprocessor (Preprocessor):
-    """Replaces double spaces at the end of the lines with <br/ >."""
-
-    def run (self, lines) :
-        for i in range(len(lines)) :
-            if (lines[i].endswith("  ")
-                and not RE.regExp['tabbed'].match(lines[i]) ):
-                lines[i] += "<br />"
-        return lines
-
-LINE_BREAKS_PREPROCESSOR = LineBreaksPreprocessor()
-
-
-class HtmlBlockPreprocessor (Preprocessor):
-    """Removes html blocks from self.lines"""
+class HtmlBlockPreprocessor(TextPreprocessor):
+    """Removes html blocks from the source text and stores it."""
     
     def _get_left_tag(self, block):
         return block[1:].replace(">", " ", 1).split()[0].lower()
         return block.rstrip()[-len(left_tag)-2:-1].lower()
 
     def _equal_tags(self, left_tag, right_tag):
-        if left_tag in ['?', '?php', 'div'] : # handle PHP, etc.
+        
+        if left_tag == 'div' or left_tag[0] in ['?', '@', '%']: # handle PHP, etc.
             return True
         if ("/" + left_tag) == right_tag:
             return True
+        if (right_tag == "--" and left_tag == "--"):
+            return True
         elif left_tag == right_tag[1:] \
             and right_tag[0] != "<":
             return True
         return (tag in ['hr', 'hr/'])
 
     
-    def run (self, lines) :
+    def run(self, text):
+
         new_blocks = []
-        text = "\n".join(lines)
         text = text.split("\n\n")
         
         items = []
         in_tag = False # flag
         
         for block in text:
-            if block.startswith("\n") :
+            if block.startswith("\n"):
                 block = block[1:]
 
             if not in_tag:
                     left_tag = self._get_left_tag(block)
                     right_tag = self._get_right_tag(left_tag, block)
 
-                    if not (is_block_level(left_tag) \
+                    if not (isBlockLevel(left_tag) \
                         or block[1] in ["!", "?", "@", "%"]):
                         new_blocks.append(block)
                         continue
                         new_blocks.append(
                             self.stash.store(block.strip()))
                         continue
-                    elif not block[1] == "!":
+                    else: #if not block[1] == "!":
                         # if is block level tag and is not complete
                         items.append(block.strip())
                         in_tag = True
                         continue
-                    
+
                 new_blocks.append(block)
 
             else:
                 items.append(block.strip())
                 
                 right_tag = self._get_right_tag(left_tag, block)
+                
                 if self._equal_tags(left_tag, right_tag):
                     # if find closing tag
                     in_tag = False
                     new_blocks.append(
                         self.stash.store('\n\n'.join(items)))
                     items = []
-                    
-        return "\n\n".join(new_blocks).split("\n")
+
+        if items:
+            new_blocks.append(self.stash.store('\n\n'.join(items)))
+            new_blocks.append('\n')
+            
+        return "\n\n".join(new_blocks)
 
 HTML_BLOCK_PREPROCESSOR = HtmlBlockPreprocessor()
 
 
-class ReferencePreprocessor (Preprocessor):
+class HeaderPreprocessor(Preprocessor):
 
-    def run (self, lines) :
+    """
+       Replaces underlined headers with hashed headers to avoid
+       the nead for lookahead later.
+    """
+
+    def run (self, lines):
+
+        i = -1
+        while i+1 < len(lines):
+            i = i+1
+            if not lines[i].strip():
+                continue
+
+            if lines[i].startswith("#"):
+                lines.insert(i+1, "\n")
+
+            if (i+1 <= len(lines)
+                  and lines[i+1]
+                  and lines[i+1][0] in ['-', '=']):
+
+                underline = lines[i+1].strip()
+
+                if underline == "="*len(underline):
+                    lines[i] = "# " + lines[i].strip()
+                    lines[i+1] = ""
+                elif underline == "-"*len(underline):
+                    lines[i] = "## " + lines[i].strip()
+                    lines[i+1] = ""
+
+        return lines
+
+HEADER_PREPROCESSOR = HeaderPreprocessor()
+
+
+class LinePreprocessor(Preprocessor):
+    """Deals with HR lines (needs to be done before processing lists)"""
+
+    blockquote_re = re.compile(r'^(> )+')
+
+    def run (self, lines):
+        for i in range(len(lines)):
+            prefix = ''
+            m = self.blockquote_re.search(lines[i])
+            if m : prefix = m.group(0)
+            if self._isLine(lines[i][len(prefix):]):
+                lines[i] = prefix + self.stash.store("<hr />", safe=True)
+        return lines
+
+    def _isLine(self, block):
+        """Determines if a block should be replaced with an <HR>"""
+        if block.startswith("    "): return 0  # a code block
+        text = "".join([x for x in block if not x.isspace()])
+        if len(text) <= 2:
+            return 0
+        for pattern in ['isline1', 'isline2', 'isline3']:
+            m = RE.regExp[pattern].match(text)
+            if (m and m.group(1)):
+                return 1
+        else:
+            return 0
+
+LINE_PREPROCESSOR = LinePreprocessor()
+
+
+class ReferencePreprocessor(Preprocessor):
+    ''' 
+    Removes reference definitions from the text and stores them for later use.
+    '''
+
+    def run (self, lines):
 
         new_text = [];
         for line in lines:
             if m:
                 id = m.group(2).strip().lower()
                 t = m.group(4).strip()  # potential title
-                if not t :
+                if not t:
                     self.references[id] = (m.group(3), t)
                 elif (len(t) >= 2
                       and (t[0] == t[-1] == "\""
                            or t[0] == t[-1] == "\'"
-                           or (t[0] == "(" and t[-1] == ")") ) ) :
+                           or (t[0] == "(" and t[-1] == ")") ) ):
                     self.references[id] = (m.group(3), t[1:-1])
-                else :
+                else:
                     new_text.append(line)
             else:
                 new_text.append(line)
 
 NOBRACKET = r'[^\]\[]*'
 BRK = ( r'\[('
-        + (NOBRACKET + r'(\['+NOBRACKET)*6
-        + (NOBRACKET+ r'\])*'+NOBRACKET)*6
+        + (NOBRACKET + r'(\[')*6
+        + (NOBRACKET+ r'\])*')*6
         + NOBRACKET + r')\]' )
+NOIMG = r'(?<!\!)'
 
 BACKTICK_RE = r'\`([^\`]*)\`'                    # `e= m*c^2`
 DOUBLE_BACKTICK_RE =  r'\`\`(.*)\`\`'            # ``e=f("`")``
 
 if SMART_EMPHASIS:
     EMPHASIS_2_RE = r'(?<!\S)_(\S[^_]*)_'        # _emphasis_
-else :
+else:
     EMPHASIS_2_RE = r'_([^_]*)_'                 # _emphasis_
 
 STRONG_2_RE = r'__([^_]*)__'                     # __strong__
 STRONG_EM_2_RE = r'___([^_]*)___'                # ___strong___
 
-LINK_RE = BRK + r'\s*\(([^\)]*)\)'               # [text](url)
-LINK_ANGLED_RE = BRK + r'\s*\(<([^\)]*)>\)'      # [text](<url>)
+LINK_RE = NOIMG + BRK + r'\s*\(([^\)]*)\)'               # [text](url)
+LINK_ANGLED_RE = NOIMG + BRK + r'\s*\(<([^\)]*)>\)'      # [text](<url>)
 IMAGE_LINK_RE = r'\!' + BRK + r'\s*\(([^\)]*)\)' # ![alttxt](http://x.com/)
-REFERENCE_RE = BRK+ r'\s*\[([^\]]*)\]'           # [Google][3]
+REFERENCE_RE = NOIMG + BRK+ r'\s*\[([^\]]*)\]'           # [Google][3]
 IMAGE_REFERENCE_RE = r'\!' + BRK + '\s*\[([^\]]*)\]' # ![alt text][2]
 NOT_STRONG_RE = r'( \* )'                        # stand-alone * or _
 AUTOLINK_RE = r'<(http://[^>]*)>'                # <http://www.123.com>
 #HTML_RE = r'(\<[^\>]*\>)'                        # <...>
 HTML_RE = r'(\<[a-zA-Z/][^\>]*\>)'               # <...>
 ENTITY_RE = r'(&[\#a-zA-Z0-9]*;)'                # &amp;
+LINE_BREAK_RE = r'  \n'                     # two spaces at end of line
+LINE_BREAK_2_RE = r'  $'                    # two spaces at end of text
 
 class Pattern:
 
-    def __init__ (self, pattern) :
+    def __init__ (self, pattern):
         self.pattern = pattern
         self.compiled_re = re.compile("^(.*)%s(.*)$" % pattern, re.DOTALL)
 
-    def getCompiledRegExp (self) :
+    def getCompiledRegExp (self):
         return self.compiled_re
 
 BasePattern = Pattern # for backward compatibility
 
-class SimpleTextPattern (Pattern) :
+class SimpleTextPattern (Pattern):
 
-    def handleMatch(self, m, doc) :
+    def handleMatch(self, m, doc):
         return doc.createTextNode(m.group(2))
 
 class SimpleTagPattern (Pattern):
 
-    def __init__ (self, pattern, tag) :
+    def __init__ (self, pattern, tag):
         Pattern.__init__(self, pattern)
         self.tag = tag
 
-    def handleMatch(self, m, doc) :
+    def handleMatch(self, m, doc):
         el = doc.createElement(self.tag)
         el.appendChild(doc.createTextNode(m.group(2)))
         return el
 
+class SubstituteTagPattern (SimpleTagPattern):
+
+    def handleMatch (self, m, doc):
+        return doc.createElement(self.tag)
+
 class BacktickPattern (Pattern):
 
     def __init__ (self, pattern):
         Pattern.__init__(self, pattern)
         self.tag = "code"
 
-    def handleMatch(self, m, doc) :
+    def handleMatch(self, m, doc):
         el = doc.createElement(self.tag)
         text = m.group(2).strip()
         #text = text.replace("&", "&amp;")
         return el
 
 
-class DoubleTagPattern (SimpleTagPattern) :
+class DoubleTagPattern (SimpleTagPattern): 
 
-    def handleMatch(self, m, doc) :
+    def handleMatch(self, m, doc):
         tag1, tag2 = self.tag.split(",")
         el1 = doc.createElement(tag1)
         el2 = doc.createElement(tag2)
 
 class HtmlPattern (Pattern):
 
-    def handleMatch (self, m, doc) :
-        place_holder = self.stash.store(m.group(2))
+    def handleMatch (self, m, doc):
+        rawhtml = m.group(2)
+        inline = True
+        place_holder = self.stash.store(rawhtml)
         return doc.createTextNode(place_holder)
 
 
 class LinkPattern (Pattern):
 
-    def handleMatch(self, m, doc) :
+    def handleMatch(self, m, doc):
         el = doc.createElement('a')
         el.appendChild(doc.createTextNode(m.group(2)))
-        parts = m.group(9).split()
+        parts = m.group(9).split('"')
         # We should now have [], [href], or [href, title]
-        if parts :
-            el.setAttribute('href', parts[0])
-        else :
+        if parts:
+            el.setAttribute('href', parts[0].strip())
+        else:
             el.setAttribute('href', "")
-        if len(parts) > 1 :
+        if len(parts) > 1:
             # we also got a title
-            title = " ".join(parts[1:]).strip()
+            title = '"' + '"'.join(parts[1:]).strip()
             title = dequote(title) #.replace('"', "&quot;")
             el.setAttribute('title', title)
         return el
     def handleMatch(self, m, doc):
         el = doc.createElement('img')
         src_parts = m.group(9).split()
-        el.setAttribute('src', src_parts[0])
-        if len(src_parts) > 1 :
+        if src_parts:
+            el.setAttribute('src', src_parts[0])
+        else:
+            el.setAttribute('src', "")
+        if len(src_parts) > 1:
             el.setAttribute('title', dequote(" ".join(src_parts[1:])))
-        if ENABLE_ATTRIBUTES :
+        if ENABLE_ATTRIBUTES:
             text = doc.createTextNode(m.group(2))
             el.appendChild(text)
             text.handleAttributes()
 class ReferencePattern (Pattern):
 
     def handleMatch(self, m, doc):
-        if m.group(9) :
+
+        if m.group(9):
             id = m.group(9).lower()
-        else :
+        else:
             # if we got something like "[Google][]"
             # we'll use "google" as the id
             id = m.group(2).lower()
-        if not self.references.has_key(id) : # ignore undefined refs
+
+        if not self.references.has_key(id): # ignore undefined refs
             return None
         href, title = self.references[id]
         text = m.group(2)
     def makeTag(self, href, title, text, doc):
         el = doc.createElement('a')
         el.setAttribute('href', href)
-        if title :
+        if title:
             el.setAttribute('title', title)
         el.appendChild(doc.createTextNode(text))
         return el
     def makeTag(self, href, title, text, doc):
         el = doc.createElement('img')
         el.setAttribute('src', href)
-        if title :
+        if title:
             el.setAttribute('title', title)
         el.setAttribute('alt', text)
         return el
 
 class AutomailPattern (Pattern):
 
-    def handleMatch(self, m, doc) :
+    def handleMatch(self, m, doc):
         el = doc.createElement('a')
         email = m.group(2)
         if email.startswith("mailto:"):
 STRONG_EM_PATTERN       = DoubleTagPattern(STRONG_EM_RE, 'strong,em')
 STRONG_EM_PATTERN_2     = DoubleTagPattern(STRONG_EM_2_RE, 'strong,em')
 
+LINE_BREAK_PATTERN      = SubstituteTagPattern(LINE_BREAK_RE, 'br ')
+LINE_BREAK_PATTERN_2    = SubstituteTagPattern(LINE_BREAK_2_RE, 'br ')
+
 LINK_PATTERN            = LinkPattern(LINK_RE)
 LINK_ANGLED_PATTERN     = LinkPattern(LINK_ANGLED_RE)
 IMAGE_LINK_PATTERN      = ImagePattern(IMAGE_LINK_RE)
 ======================================================================
 
 Markdown also allows post-processors, which are similar to
-preprocessors in that they need to implement a "run" method.  Unlike
-pre-processors, they take a NanoDom document as a parameter and work
-with that.
+preprocessors in that they need to implement a "run" method. However,
+they are run after core processing.
 
-Post-Processor should extend markdown.Postprocessor.
-
-There are currently no standard post-processors, but the footnote
-extension below uses one.
+There are two types of post-processors: Postprocessor and TextPostprocessor
 """
 
-class Postprocessor :
-    pass
 
+class Postprocessor:
+    '''
+    Postprocessors are run before the dom it converted back into text.
+    
+    Each Postprocessor implements a "run" method that takes a pointer to a
+    NanoDom document, modifies it as necessary and returns a NanoDom 
+    document.
+    
+    Postprocessors must extend markdown.Postprocessor.
+
+    There are currently no standard post-processors, but the footnote
+    extension uses one.
+    '''
+
+    def run(self, dom):
+        pass
+
+
+
+class TextPostprocessor:
+    '''
+    TextPostprocessors are run after the dom it converted back into text.
+    
+    Each TextPostprocessor implements a "run" method that takes a pointer to a
+    text string, modifies it as necessary and returns a text string.
+    
+    TextPostprocessors must extend markdown.TextPostprocessor.
+    '''
+
+    def run(self, text):
+        pass
+
+
+class RawHtmlTextPostprocessor(TextPostprocessor):
+
+    def __init__(self):
+        pass
+
+    def run(self, text):
+        for i in range(self.stash.html_counter):
+            html, safe  = self.stash.rawHtmlBlocks[i]
+            if self.safeMode and not safe:
+                if str(self.safeMode).lower() == 'escape':
+                    html = self.escape(html)
+                elif str(self.safeMode).lower() == 'remove':
+                    html = ''
+                else:
+                    html = HTML_REMOVED_TEXT
+                                   
+            text = text.replace("<p>%s\n</p>" % (HTML_PLACEHOLDER % i),
+                              html + "\n")
+            text =  text.replace(HTML_PLACEHOLDER % i, html)
+        return text
+
+    def escape(self, html):
+        ''' Basic html escaping '''
+        html = html.replace('&', '&amp;')
+        html = html.replace('<', '&lt;')
+        html = html.replace('>', '&gt;')
+        return html.replace('"', '&quot;')
+
+RAWHTMLTEXTPOSTPROCESSOR = RawHtmlTextPostprocessor()
 
 """
 ======================================================================
 ======================================================================
 """
 
-class HtmlStash :
+class HtmlStash:
     """This class is used for stashing HTML objects that we extract
         in the beginning and replace with place-holders."""
 
-    def __init__ (self) :
+    def __init__ (self):
         self.html_counter = 0 # for counting inline html segments
         self.rawHtmlBlocks=[]
 
-    def store(self, html) :
+    def store(self, html, safe=False):
         """Saves an HTML segment for later reinsertion.  Returns a
            placeholder string that needs to be inserted into the
            document.
 
            @param html: an html segment
+           @param safe: label an html segment as safe for safemode
+           @param inline: label a segmant as inline html
            @returns : a placeholder string """
-        self.rawHtmlBlocks.append(html)
+        self.rawHtmlBlocks.append((html, safe))
         placeholder = HTML_PLACEHOLDER % self.html_counter
         self.html_counter += 1
         return placeholder
 
 
-class BlockGuru :
+class BlockGuru:
 
-    def _findHead(self, lines, fn, allowBlank=0) :
+    def _findHead(self, lines, fn, allowBlank=0):
 
         """Functional magic to help determine boundaries of indented
            blocks.
 
         i = 0 # to keep track of where we are
 
-        for line in lines :
+        for line in lines:
 
             if not line.strip() and not allowBlank:
                 return items, lines[i:]
                 i += 1
 
                 # Find the next non-blank line
-                for j in range(i, len(lines)) :
-                    if lines[j].strip() :
+                for j in range(i, len(lines)):
+                    if lines[j].strip():
                         next = lines[j]
                         break
-                else :
+                else:
                     # There is no more text => this is the end
                     break
 
 
                 part = fn(next)
 
-                if part :
+                if part:
                     items.append("")
                     continue
-                else :
+                else:
                     break # found end of the list
 
             part = fn(line)
 
-            if part :
+            if part:
                 items.append(part)
                 i += 1
                 continue
-            else :
+            else:
                 return items, lines[i:]
-        else :
+        else:
             i += 1
 
         return items, lines[i:]
 
 
-    def detabbed_fn(self, line) :
+    def detabbed_fn(self, line):
         """ An auxiliary method to be passed to _findHead """
         m = RE.regExp['tabbed'].match(line)
         if m:
             return m.group(4)
-        else :
+        else:
             return None
 
 
-    def detectTabbed(self, lines) :
+    def detectTabbed(self, lines):
 
         return self._findHead(lines, self.detabbed_fn,
                               allowBlank = 1)
     sys.stderr.write(string +'\n')
 
 
-def dequote(string) :
+def dequote(string):
     """ Removes quotes from around a string """
     if ( ( string.startswith('"') and string.endswith('"'))
-         or (string.startswith("'") and string.endswith("'")) ) :
+         or (string.startswith("'") and string.endswith("'")) ):
         return string[1:-1]
-    else :
+    else:
         return string
 
 """
 inline patterns or a combination of the three.
 """
 
-class CorePatterns :
+class CorePatterns:
     """This class is scheduled for removal as part of a refactoring
         effort."""
 
     patterns = {
         'header':          r'(#*)([^#]*)(#*)', # # A title
-        'reference-def' :  r'(\ ?\ ?\ ?)\[([^\]]*)\]:\s*([^ ]*)(.*)',
+        'reference-def':   r'(\ ?\ ?\ ?)\[([^\]]*)\]:\s*([^ ]*)(.*)',
                            # [Google]: http://www.google.com/
         'containsline':    r'([-]*)$|^([=]*)', # -----, =====, etc.
         'ol':              r'[ ]{0,3}[\d]*\.\s+(.*)', # 1. text
         'isline2':         r'(\-*)', # ---
         'isline3':         r'(\_*)', # ___
         'tabbed':          r'((\t)|(    ))(.*)', # an indented line
-        'quoted' :         r'> ?(.*)', # a quoted block ("> ...")
+        'quoted':          r'> ?(.*)', # a quoted block ("> ...")
     }
 
-    def __init__ (self) :
+    def __init__ (self):
 
         self.regExp = {}
-        for key in self.patterns.keys() :
+        for key in self.patterns.keys():
             self.regExp[key] = re.compile("^%s$" % self.patterns[key],
                                           re.DOTALL)
 
         Markdown text """
 
 
-    def __init__(self, source=None,
+    def __init__(self, source=None,  # depreciated
                  extensions=[],
                  extension_configs=None,
-                 encoding=None,
-                 safe_mode = True):
+                 safe_mode = False):
         """Creates a new Markdown instance.
 
-           @param source: The text in Markdown format.
-           @param encoding: The character encoding of <text>. """
+           @param source: The text in Markdown format. Depreciated!
+           @param extensions: A list if extensions.
+           @param extension-configs: Configuration setting for extensions.
+           @param safe_mode: Disallow raw html. """
 
+        self.source = source
+        if source is not None:
+            message(WARN, "The `source` arg of Markdown.__init__() is depreciated and will be removed in the future. Use `instance.convert(source)` instead.")
         self.safeMode = safe_mode
-        self.encoding = encoding
-        self.source = source
         self.blockGuru = BlockGuru()
         self.registeredExtensions = []
         self.stripTopLevelTags = 1
         self.docType = ""
 
-        self.preprocessors = [ HEADER_PREPROCESSOR,
-                               LINE_PREPROCESSOR,
-                               HTML_BLOCK_PREPROCESSOR,
-                               LINE_BREAKS_PREPROCESSOR,
-                               # A footnote preprocessor will
-                               # get inserted here
-                               REFERENCE_PREPROCESSOR ]
+        self.textPreprocessors = [HTML_BLOCK_PREPROCESSOR]
+
+        self.preprocessors = [HEADER_PREPROCESSOR,
+                              LINE_PREPROCESSOR,
+                              # A footnote preprocessor will
+                              # get inserted here
+                              REFERENCE_PREPROCESSOR]
 
 
         self.postprocessors = [] # a footnote postprocessor will get
                                  # inserted later
 
-        self.textPostprocessors = [] # a footnote postprocessor will get
-                                     # inserted later                                 
+        self.textPostprocessors = [# a footnote postprocessor will get
+                                   # inserted here
+                                   RAWHTMLTEXTPOSTPROCESSOR]
 
         self.prePatterns = []
         
 
-        self.inlinePatterns = [ DOUBLE_BACKTICK_PATTERN,
-                                BACKTICK_PATTERN,
-                                ESCAPE_PATTERN,
-                                IMAGE_LINK_PATTERN,
-                                IMAGE_REFERENCE_PATTERN,
-                                REFERENCE_PATTERN,
-                                LINK_ANGLED_PATTERN,
-                                LINK_PATTERN,
-                                AUTOLINK_PATTERN,
-                                AUTOMAIL_PATTERN,
-                                HTML_PATTERN,
-                                ENTITY_PATTERN,
-                                NOT_STRONG_PATTERN,
-                                STRONG_EM_PATTERN,
-                                STRONG_EM_PATTERN_2,
-                                STRONG_PATTERN,
-                                STRONG_PATTERN_2,
-                                EMPHASIS_PATTERN,
-                                EMPHASIS_PATTERN_2
-                                # The order of the handlers matters!!!
-                                ]
+        self.inlinePatterns = [DOUBLE_BACKTICK_PATTERN,
+                               BACKTICK_PATTERN,
+                               ESCAPE_PATTERN,
+                               REFERENCE_PATTERN,
+                               LINK_ANGLED_PATTERN,
+                               LINK_PATTERN,
+                               IMAGE_LINK_PATTERN,
+			                   IMAGE_REFERENCE_PATTERN,
+			                   AUTOLINK_PATTERN,
+                               AUTOMAIL_PATTERN,
+                               LINE_BREAK_PATTERN_2,
+                               LINE_BREAK_PATTERN,
+                               HTML_PATTERN,
+                               ENTITY_PATTERN,
+                               NOT_STRONG_PATTERN,
+                               STRONG_EM_PATTERN,
+                               STRONG_EM_PATTERN_2,
+                               STRONG_PATTERN,
+                               STRONG_PATTERN_2,
+                               EMPHASIS_PATTERN,
+                               EMPHASIS_PATTERN_2
+                               # The order of the handlers matters!!!
+                               ]
 
         self.registerExtensions(extensions = extensions,
                                 configs = extension_configs)
         self.reset()
 
 
-    def registerExtensions(self, extensions, configs) :
+    def registerExtensions(self, extensions, configs):
 
-        if not configs :
+        if not configs:
             configs = {}
 
-        for ext in extensions :
+        for ext in extensions:
 
             extension_module_name = "mdx_" + ext
 
-            try :
+            try:
                 module = __import__(extension_module_name)
 
-            except :
+            except:
                 message(CRITICAL,
                         "couldn't load extension %s (looking for %s module)"
                         % (ext, extension_module_name) )
-            else :
+            else:
 
-                if configs.has_key(ext) :
+                if configs.has_key(ext):
                     configs_for_ext = configs[ext]
-                else :
+                else:
                     configs_for_ext = []
                 extension = module.makeExtension(configs_for_ext)    
                 extension.extendMarkdown(self, globals())
 
 
 
-    def registerExtension(self, extension) :
+    def registerExtension(self, extension):
         """ This gets called by the extension """
         self.registeredExtensions.append(extension)
 
-    def reset(self) :
+    def reset(self):
         """Resets all state variables so that we can start
             with a new text."""
         self.references={}
         self.htmlStash = HtmlStash()
 
         HTML_BLOCK_PREPROCESSOR.stash = self.htmlStash
+        LINE_PREPROCESSOR.stash = self.htmlStash
         REFERENCE_PREPROCESSOR.references = self.references
         HTML_PATTERN.stash = self.htmlStash
         ENTITY_PATTERN.stash = self.htmlStash
         REFERENCE_PATTERN.references = self.references
         IMAGE_REFERENCE_PATTERN.references = self.references
+        RAWHTMLTEXTPOSTPROCESSOR.stash = self.htmlStash
+        RAWHTMLTEXTPOSTPROCESSOR.safeMode = self.safeMode
 
-        for extension in self.registeredExtensions :
+        for extension in self.registeredExtensions:
             extension.reset()
 
 
         self.doc.appendChild(self.top_element)
 
         # Fixup the source text
-        text = self.source.strip()
+        text = self.source
         text = text.replace("\r\n", "\n").replace("\r", "\n")
         text += "\n\n"
         text = text.expandtabs(TAB_LENGTH)
 
 
         buffer = []
-        for line in self.lines :
-            if line.startswith("#") :
+        for line in self.lines:
+            if line.startswith("#"):
                 self._processSection(self.top_element, buffer)
                 buffer = [line]
-            else :
+            else:
                 buffer.append(line)
         self._processSection(self.top_element, buffer)
         
         self.top_element.appendChild(self.doc.createTextNode('\n'))
 
         # Run the post-processors
-        for postprocessor in self.postprocessors :
+        for postprocessor in self.postprocessors:
             postprocessor.run(self.doc)
 
         return self.doc
 
 
     def _processSection(self, parent_elem, lines,
-                        inList = 0, looseList = 0) :
+                        inList = 0, looseList = 0):
 
         """Process a section of a source document, looking for high
            level structural elements like lists, block quotes, code
            @param inList: a level
            @returns: None"""
 
-        if not lines :
-            return
+        # Loop through lines until none left.
+        while lines:
 
-        # Check if this section starts with a list, a blockquote or
-        # a code block
+            # Check if this section starts with a list, a blockquote or
+            # a code block
 
-        processFn = { 'ul' :     self._processUList,
-                      'ol' :     self._processOList,
-                      'quoted' : self._processQuote,
-                      'tabbed' : self._processCodeBlock }
+            processFn = { 'ul':     self._processUList,
+                          'ol':     self._processOList,
+                          'quoted': self._processQuote,
+                          'tabbed': self._processCodeBlock}
 
-        for regexp in ['ul', 'ol', 'quoted', 'tabbed'] :
-            m = RE.regExp[regexp].match(lines[0])
-            if m :
-                processFn[regexp](parent_elem, lines, inList)
-                return
+            for regexp in ['ul', 'ol', 'quoted', 'tabbed']:
+                m = RE.regExp[regexp].match(lines[0])
+                if m:
+                    processFn[regexp](parent_elem, lines, inList)
+                    return
 
-        # We are NOT looking at one of the high-level structures like
-        # lists or blockquotes.  So, it's just a regular paragraph
-        # (though perhaps nested inside a list or something else).  If
-        # we are NOT inside a list, we just need to look for a blank
-        # line to find the end of the block.  If we ARE inside a
-        # list, however, we need to consider that a sublist does not
-        # need to be separated by a blank line.  Rather, the following
-        # markup is legal:
-        #
-        # * The top level list item
-        #
-        #     Another paragraph of the list.  This is where we are now.
-        #     * Underneath we might have a sublist.
-        #
+            # We are NOT looking at one of the high-level structures like
+            # lists or blockquotes.  So, it's just a regular paragraph
+            # (though perhaps nested inside a list or something else).  If
+            # we are NOT inside a list, we just need to look for a blank
+            # line to find the end of the block.  If we ARE inside a
+            # list, however, we need to consider that a sublist does not
+            # need to be separated by a blank line.  Rather, the following
+            # markup is legal:
+            #
+            # * The top level list item
+            #
+            #     Another paragraph of the list.  This is where we are now.
+            #     * Underneath we might have a sublist.
+            #
 
-        if inList :
+            if inList:
 
-            start, theRest = self._linesUntil(lines, (lambda line:
-                             RE.regExp['ul'].match(line)
-                             or RE.regExp['ol'].match(line)
-                                              or not line.strip()))
+                start, lines  = self._linesUntil(lines, (lambda line:
+                                 RE.regExp['ul'].match(line)
+                                 or RE.regExp['ol'].match(line)
+                                                  or not line.strip()))
 
-            self._processSection(parent_elem, start,
-                                 inList - 1, looseList = looseList)
-            self._processSection(parent_elem, theRest,
-                                 inList - 1, looseList = looseList)
+                self._processSection(parent_elem, start,
+                                     inList - 1, looseList = looseList)
+                inList = inList-1
 
+            else: # Ok, so it's just a simple block
 
-        else : # Ok, so it's just a simple block
+                paragraph, lines = self._linesUntil(lines, lambda line:
+                                                     not line.strip())
 
-            paragraph, theRest = self._linesUntil(lines, lambda line:
-                                                 not line.strip())
+                if len(paragraph) and paragraph[0].startswith('#'):
+                    self._processHeader(parent_elem, paragraph)
 
-            if len(paragraph) and paragraph[0].startswith('#') :
-                m = RE.regExp['header'].match(paragraph[0])
-                if m :
-                    level = len(m.group(1))
-                    h = self.doc.createElement("h%d" % level)
-                    parent_elem.appendChild(h)
-                    for item in self._handleInlineWrapper2(m.group(2).strip()) :
-                        h.appendChild(item)
-                else :
-                    message(CRITICAL, "We've got a problem header!")
+                elif paragraph:
+                    self._processParagraph(parent_elem, paragraph,
+                                          inList, looseList)
 
-            elif paragraph :
+            if lines and not lines[0].strip():
+                lines = lines[1:]  # skip the first (blank) line
 
-                list = self._handleInlineWrapper2("\n".join(paragraph))
 
-                if ( parent_elem.nodeName == 'li'
-                     and not (looseList or parent_elem.childNodes)):
+    def _processHeader(self, parent_elem, paragraph):
+        m = RE.regExp['header'].match(paragraph[0])
+        if m:
+            level = len(m.group(1))
+            h = self.doc.createElement("h%d" % level)
+            parent_elem.appendChild(h)
+            for item in self._handleInline(m.group(2).strip()):
+                h.appendChild(item)
+        else:
+            message(CRITICAL, "We've got a problem header!")
 
-                    #and not parent_elem.childNodes) :
-                    # If this is the first paragraph inside "li", don't
-                    # put <p> around it - append the paragraph bits directly
-                    # onto parent_elem
-                    el = parent_elem
-                else :
-                    # Otherwise make a "p" element
-                    el = self.doc.createElement("p")
-                    parent_elem.appendChild(el)
 
-                for item in list :
-                    el.appendChild(item)
+    def _processParagraph(self, parent_elem, paragraph, inList, looseList):
+        list = self._handleInline("\n".join(paragraph))
 
-            if theRest :
-                theRest = theRest[1:]  # skip the first (blank) line
+        if ( parent_elem.nodeName == 'li'
+                and not (looseList or parent_elem.childNodes)):
 
-            self._processSection(parent_elem, theRest, inList)
+            # If this is the first paragraph inside "li", don't
+            # put <p> around it - append the paragraph bits directly
+            # onto parent_elem
+            el = parent_elem
+        else:
+            # Otherwise make a "p" element
+            el = self.doc.createElement("p")
+            parent_elem.appendChild(el)
 
+        for item in list:
+            el.appendChild(item)
+ 
 
-
-    def _processUList(self, parent_elem, lines, inList) :
+    def _processUList(self, parent_elem, lines, inList):
         self._processList(parent_elem, lines, inList,
                          listexpr='ul', tag = 'ul')
 
-    def _processOList(self, parent_elem, lines, inList) :
+    def _processOList(self, parent_elem, lines, inList):
         self._processList(parent_elem, lines, inList,
                          listexpr='ol', tag = 'ol')
 
 
-    def _processList(self, parent_elem, lines, inList, listexpr, tag) :
+    def _processList(self, parent_elem, lines, inList, listexpr, tag):
         """Given a list of document lines starting with a list item,
            finds the end of the list, breaks it up, and recursively
            processes each list item and the remainder of the text file.
 
         i = 0  # a counter to keep track of where we are
 
-        for line in lines :
+        for line in lines: 
 
             loose = 0
-            if not line.strip() :
+            if not line.strip():
                 # If we see a blank line, this _might_ be the end of the list
                 i += 1
                 loose = 1
 
                 # Find the next non-blank line
-                for j in range(i, len(lines)) :
-                    if lines[j].strip() :
+                for j in range(i, len(lines)):
+                    if lines[j].strip():
                         next = lines[j]
                         break
-                else :
+                else:
                     # There is no more text => end of the list
                     break
 
                     items[item].append(line.strip())
                     looseList = loose or looseList
                     continue
-                else :
+                else:
                     break # found end of the list
 
             # Now we need to detect list items (at the current level)
             for expr in ['ul', 'ol', 'tabbed']:
 
                 m = RE.regExp[expr].match(line)
-                if m :
-                    if expr in ['ul', 'ol'] :  # We are looking at a new item
-                        if m.group(1) :
-                            items.append([m.group(1)])
-                            item += 1
-                    elif expr == 'tabbed' :  # This line needs to be detabbed
+                if m:
+                    if expr in ['ul', 'ol']:  # We are looking at a new item
+                        #if m.group(1) :
+                        # Removed the check to allow for a blank line
+                        # at the beginning of the list item
+                        items.append([m.group(1)])
+                        item += 1
+                    elif expr == 'tabbed':  # This line needs to be detabbed
                         items[item].append(m.group(4)) #after the 'tab'
 
                     i += 1
                     break
-            else :
+            else:
                 items[item].append(line)  # Just regular continuation
                 i += 1 # added on 2006.02.25
-        else :
+        else:
             i += 1
 
         # Add the dom elements
-        for item in items :
+        for item in items:
             li = self.doc.createElement("li")
             ul.appendChild(li)
 
         self._processSection(parent_elem, lines[i:], inList)
 
 
-    def _linesUntil(self, lines, condition) :
+    def _linesUntil(self, lines, condition):
         """ A utility function to break a list of lines upon the
             first line that satisfied a condition.  The condition
             argument should be a predicate function.
             """
 
         i = -1
-        for line in lines :
+        for line in lines:
             i += 1
-            if condition(line) : break
-        else :
+            if condition(line): break
+        else:
             i += 1
         return lines[:i], lines[i:]
 
-    def _processQuote(self, parent_elem, lines, inList) :
+    def _processQuote(self, parent_elem, lines, inList):
         """Given a list of document lines starting with a quote finds
            the end of the quote, unindents it and recursively
            processes the body of the quote and the remainder of the
 
         dequoted = []
         i = 0
-        for line in lines :
+        blank_line = False # allow one blank line between paragraphs
+        for line in lines:
             m = RE.regExp['quoted'].match(line)
-            if m :
+            if m:
                 dequoted.append(m.group(1))
                 i += 1
-            else :
+                blank_line = False
+            elif not blank_line and line.strip() != '':
+                dequoted.append(line)
+                i += 1
+            elif not blank_line and line.strip() == '':
+                dequoted.append(line)
+                i += 1
+                blank_line = True
+            else:
                 break
-        else :
-            i += 1
 
         blockquote = self.doc.createElement('blockquote')
         parent_elem.appendChild(blockquote)
 
 
 
-    def _processCodeBlock(self, parent_elem, lines, inList) :
+    def _processCodeBlock(self, parent_elem, lines, inList):
         """Given a list of document lines starting with a code block
            finds the end of the block, puts it into the dom verbatim
            wrapped in ("<pre><code>") and recursively processes the
         self._processSection(parent_elem, theRest, inList)
 
 
-    def _handleInlineWrapper2 (self, line) :
 
-
-        parts = [line]
-
-        #if not(line):
-        #    return [self.doc.createTextNode(' ')]
-
-        for pattern in self.inlinePatterns :
-
-            #print
-            #print self.inlinePatterns.index(pattern)
-
-            i = 0
-
-            #print parts
-            while i < len(parts) :
-                
-                x = parts[i]
-                #print i
-                if isinstance(x, (str, unicode)) :
-                    result = self._applyPattern(x, pattern)
-                    #print result
-                    #print result
-                    #print parts, i
-                    if result :
-                        i -= 1
-                        parts.remove(x)
-                        for y in result :
-                            parts.insert(i+1,y)
-                
-                i += 1
-
-        for i in range(len(parts)) :
-            x = parts[i]
-            if isinstance(x, (str, unicode)) :
-                parts[i] = self.doc.createTextNode(x)
-
-        return parts
-        
-
-
-    def _handleInlineWrapper (self, line) :
-
-        # A wrapper around _handleInline to avoid recursion
-
-        parts = [line]
-
-        i = 0
-        
-        while i < len(parts) :
-            x = parts[i]
-            if isinstance(x, (str, unicode)) :
-                parts.remove(x)
-                result = self._handleInline(x)
-                for y in result :
-                    parts.insert(i,y)
-            else :
-                i += 1
-
-        return parts
-
-    def _handleInline(self,  line):
+    def _handleInline (self, line, patternIndex=0):
         """Transform a Markdown line with inline elements to an XHTML
         fragment.
 
         This function uses auxiliary objects called inline patterns.
         See notes on inline patterns above.
 
-        @param item: A block of Markdown text
+        @param line: A line of Markdown text
+        @param patternIndex: The index of the inlinePattern to start with
         @return: A list of NanoDom nodes """
 
-        if not(line):
-            return [self.doc.createTextNode(' ')]
 
-        for pattern in self.inlinePatterns :
-            list = self._applyPattern( line, pattern)
-            if list: return list
+        parts = [line]
 
-        return [self.doc.createTextNode(line)]
+        while patternIndex < len(self.inlinePatterns):
 
-    def _applyPattern(self, line, pattern) :
+            i = 0
+
+            while i < len(parts):
+                
+                x = parts[i]
+
+                if isinstance(x, (str, unicode)):
+                    result = self._applyPattern(x, \
+                                self.inlinePatterns[patternIndex], \
+                                patternIndex)
+
+                    if result:
+                        i -= 1
+                        parts.remove(x)
+                        for y in result:
+                            parts.insert(i+1,y)
+
+                i += 1
+            patternIndex += 1
+
+        for i in range(len(parts)):
+            x = parts[i]
+            if isinstance(x, (str, unicode)):
+                parts[i] = self.doc.createTextNode(x)
+
+        return parts
+        
+
+    def _applyPattern(self, line, pattern, patternIndex):
+
         """ Given a pattern name, this function checks if the line
         fits the pattern, creates the necessary elements, and returns
         back a list consisting of NanoDom elements and/or strings.
         # match the line to pattern's pre-compiled reg exp.
         # if no match, move on.
 
+
+
         m = pattern.getCompiledRegExp().match(line)
-        if not m :
+        if not m:
             return None
 
         # if we got a match let the pattern make us a NanoDom node
         # if it doesn't, move on
         node = pattern.handleMatch(m, self.doc)
 
-        if node :
+        # check if any of this nodes have children that need processing
+
+        if isinstance(node, Element):
+
+            if not node.nodeName in ["code", "pre"]:
+                for child in node.childNodes:
+                    if isinstance(child, TextNode):
+                        
+                        result = self._handleInline(child.value, patternIndex+1)
+                        
+                        if result:
+
+                            if result == [child]:
+                                continue
+                                
+                            result.reverse()
+                            #to make insertion easier
+
+                            position = node.childNodes.index(child)
+                            
+                            node.removeChild(child)
+
+                            for item in result:
+
+                                if isinstance(item, (str, unicode)):
+                                    if len(item) > 0:
+                                        node.insertChild(position,
+                                             self.doc.createTextNode(item))
+                                else:
+                                    node.insertChild(position, item)
+                
+
+
+
+        if node:
             # Those are in the reverse order!
             return ( m.groups()[-1], # the string to the left
                      node,           # the new node
                      m.group(1))     # the string to the right of the match
 
-        else :
+        else:
             return None
 
-    def __str__(self, source = None):
+    def convert (self, source = None):
         """Return the document in XHTML format.
 
         @returns: A serialized XHTML body."""
-        #try :
 
-        if source :
+        if source is not None: #Allow blank string
             self.source = source
-        
+
+        if not self.source:
+            return u""
+
+        try:
+            self.source = unicode(self.source)
+        except UnicodeDecodeError:
+            message(CRITICAL, 'UnicodeDecodeError: Markdown only accepts unicode or ascii  input.')
+            return u""
+
+        for pp in self.textPreprocessors:
+            self.source = pp.run(self.source)
+
         doc = self._transform()
         xml = doc.toxml()
 
-        #finally:
-        #    doc.unlink()
 
-        # Let's stick in all the raw html pieces
+        # Return everything but the top level tag
 
-        for i in range(self.htmlStash.html_counter) :
-            html = self.htmlStash.rawHtmlBlocks[i]
-            if self.safeMode :
-                html = "[HTML_REMOVED]"
-                
-            xml = xml.replace("<p>%s\n</p>" % (HTML_PLACEHOLDER % i),
-                              html + "\n")
-            xml = xml.replace(HTML_PLACEHOLDER % i,
-                              html)
-
-        # And return everything but the top level tag
-
-        if self.stripTopLevelTags :
+        if self.stripTopLevelTags:
             xml = xml.strip()[23:-7] + "\n"
 
-        for pp in self.textPostprocessors :
+        for pp in self.textPostprocessors:
             xml = pp.run(xml)
 
-        return self.docType + xml
+        return (self.docType + xml).strip()
 
 
-    toString = __str__
+    def __str__(self):
+        ''' Report info about instance. Markdown always returns unicode. '''
+        if self.source is None:
+            status = 'in which no source text has been assinged.'
+        else:
+            status = 'which contains %d chars and %d line(s) of source.'%\
+                     (len(self.source), self.source.count('\n')+1)
+        return 'An instance of "%s" %s'% (self.__class__, status)
 
+    __unicode__ = convert # markdown should always return a unicode string
 
-    def __unicode__(self):
-        """Return the document in XHTML format as a Unicode object.
-        """
-        return str(self)#.decode(self.encoding)
-
-
-    toUnicode = __unicode__
 
 
 
                      extensions = [],
                      encoding = None,
                      message_threshold = CRITICAL,
-                     safe = False) :
+                     safe = False):
 
-    global MESSAGE_THRESHOLD
-    MESSAGE_THRESHOLD = message_threshold
+    global console_hndlr
+    console_hndlr.setLevel(message_threshold)
 
-    message(VERBOSE, "input file: %s" % input)
+    message(DEBUG, "input file: %s" % input)
 
-
-    if not encoding :
+    if not encoding:
         encoding = "utf-8"
 
-    input_file = codecs.open(input, mode="r", encoding="utf-8")
+    input_file = codecs.open(input, mode="r", encoding=encoding)
     text = input_file.read()
     input_file.close()
 
-    new_text = markdown(text, extensions, encoding, safe_mode = safe)
+    text = removeBOM(text, encoding)
 
-    if output :
+    new_text = markdown(text, extensions, safe_mode = safe)
+
+    if output:
         output_file = codecs.open(output, "w", encoding=encoding)
         output_file.write(new_text)
         output_file.close()
 
-    else :
+    else:
         sys.stdout.write(new_text.encode(encoding))
 
 def markdown(text,
              extensions = [],
-             encoding = None,
-             safe_mode = False) :
+             safe_mode = False):
     
-    message(VERBOSE, "in markdown.markdown(), received text:\n%s" % text)
+    message(DEBUG, "in markdown.markdown(), received text:\n%s" % text)
 
     extension_names = []
     extension_configs = {}
     
-    for ext in extensions :
+    for ext in extensions:
         pos = ext.find("(") 
-        if pos == -1 :
+        if pos == -1:
             extension_names.append(ext)
-        else :
+        else:
             name = ext[:pos]
             extension_names.append(name)
             pairs = [x.split("=") for x in ext[pos+1:-1].split(",")]
             configs = [(x.strip(), y.strip()) for (x, y) in pairs]
             extension_configs[name] = configs
-            #print configs
 
-    md = Markdown(text, extensions=extension_names,
+    md = Markdown(extensions=extension_names,
                   extension_configs=extension_configs,
                   safe_mode = safe_mode)
 
-    return md.toString()
+    return md.convert(text)
         
 
-class Extension :
+class Extension:
 
-    def __init__(self, configs = {}) :
+    def __init__(self, configs = {}):
         self.config = configs
 
-    def getConfig(self, key) :
-        if self.config.has_key(key) :
-            #print self.config[key][0]
+    def getConfig(self, key):
+        if self.config.has_key(key):
             return self.config[key][0]
-        else :
+        else:
             return ""
 
-    def getConfigInfo(self) :
+    def getConfigInfo(self):
         return [(key, self.config[key][1]) for key in self.config.keys()]
 
-    def setConfig(self, key, value) :
+    def setConfig(self, key, value):
         self.config[key][0] = value
 
 
     
 """ % EXECUTABLE_NAME_FOR_USAGE
 
-def parse_options() :
+def parse_options():
 
-    try :
+    try:
         optparse = __import__("optparse")
-    except :
-        if len(sys.argv) == 2 :
-            return {'input' : sys.argv[1],
-                    'output' : None,
-                    'message_threshold' : CRITICAL,
-                    'safe' : False,
-                    'extensions' : [],
-                    'encoding' : None }
+    except:
+        if len(sys.argv) == 2:
+            return {'input': sys.argv[1],
+                    'output': None,
+                    'message_threshold': CRITICAL,
+                    'safe': False,
+                    'extensions': [],
+                    'encoding': None }
 
-        else :
+        else:
             print OPTPARSE_WARNING
             return None
 
     parser.add_option("-e", "--encoding", dest="encoding",
                       help="encoding for input and output files",)
     parser.add_option("-q", "--quiet", default = CRITICAL,
-                      action="store_const", const=NONE, dest="verbose",
+                      action="store_const", const=60, dest="verbose",
                       help="suppress all messages")
     parser.add_option("-v", "--verbose",
                       action="store_const", const=INFO, dest="verbose",
                       help="print info messages")
-    parser.add_option("-s", "--safe",
-                      action="store_const", const=True, dest="safe",
-                      help="same mode (strip user's HTML tag)")
+    parser.add_option("-s", "--safe", dest="safe", default=False,
+                      metavar="SAFE_MODE",
+                      help="same mode ('replace', 'remove' or 'escape'  user's HTML tag)")
     
     parser.add_option("--noisy",
-                      action="store_const", const=VERBOSE, dest="verbose",
+                      action="store_const", const=DEBUG, dest="verbose",
                       help="print debug messages")
     parser.add_option("-x", "--extension", action="append", dest="extensions",
                       help = "load extension EXTENSION", metavar="EXTENSION")
 
     (options, args) = parser.parse_args()