Anonymous avatar Anonymous committed 32c463b

- refactor template
- install pyments and gasessions

Comments (0)

Files changed (100)

+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>meblog</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.python.pydev.PyDevBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.python.pydev.pythonNature</nature>
+	</natures>
+</projectDescription>
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?eclipse-pydev version="1.0"?>
+
+<pydev_project>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.5</pydev_property>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
+</pydev_project>

context/models.py

-from appengine_django.models import BaseModel
-from google.appengine.ext import db
-
-# Create your models here.
Add a comment to this file

context/views.py

Empty file removed.

gaesessions/__init__.py

+"""A fast, lightweight, and secure session WSGI middleware for use with GAE."""
+from Cookie import CookieError, SimpleCookie
+from base64 import b64decode, b64encode
+import datetime
+import hashlib
+import hmac
+import logging
+import pickle
+import os
+import time
+
+from google.appengine.api import memcache
+from google.appengine.ext import db
+
+# Configurable cookie options
+COOKIE_NAME_PREFIX = "DgU"  # identifies a cookie as being one used by gae-sessions (so you can set cookies too)
+COOKIE_PATH = "/"
+DEFAULT_COOKIE_ONLY_THRESH = 10240  # 10KB: GAE only allows ~16000B in HTTP header - leave ~6KB for other info
+DEFAULT_LIFETIME = datetime.timedelta(days=7)
+
+# constants
+SID_LEN = 43  # timestamp (10 chars) + underscore + md5 (32 hex chars)
+SIG_LEN = 44  # base 64 encoded HMAC-SHA256
+MAX_COOKIE_LEN = 4096
+EXPIRE_COOKIE_FMT = ' %s=; expires=Wed, 01-Jan-1970 00:00:00 GMT; Path=' + COOKIE_PATH
+COOKIE_FMT = ' ' + COOKIE_NAME_PREFIX + '%02d="%s"; expires=%s; Path=' + COOKIE_PATH + '; HttpOnly'
+COOKIE_FMT_SECURE = COOKIE_FMT + '; Secure'
+COOKIE_DATE_FMT = '%a, %d-%b-%Y %H:%M:%S GMT'
+COOKIE_OVERHEAD = len(COOKIE_FMT % (0, '', '')) + 29 + 150  # 29=date len, 150=safety margin (e.g., in case browser uses 4000 instead of 4096)
+MAX_DATA_PER_COOKIE = MAX_COOKIE_LEN - COOKIE_OVERHEAD
+
+_current_session = None
+def get_current_session():
+    """Returns the session associated with the current request."""
+    return _current_session
+
+def is_gaesessions_key(k):
+    return k.startswith(COOKIE_NAME_PREFIX)
+
+class SessionModel(db.Model):
+    """Contains session data.  key_name is the session ID and pdump contains a
+    pickled dictionary which maps session variables to their values."""
+    pdump = db.BlobProperty()
+
+class Session(object):
+    """Manages loading, reading/writing key-value pairs, and saving of a session.
+
+    ``sid`` - if set, then the session for that sid (if any) is loaded. Otherwise,
+    sid will be loaded from the HTTP_COOKIE (if any).
+    """
+    DIRTY_BUT_DONT_PERSIST_TO_DB = 1
+
+    def __init__(self, sid=None, lifetime=DEFAULT_LIFETIME, no_datastore=False,
+                 cookie_only_threshold=DEFAULT_COOKIE_ONLY_THRESH, cookie_key=None):
+        self.sid = None
+        self.cookie_keys = []
+        self.cookie_data = None
+        self.data = {}
+        self.dirty = False  # has the session been changed?
+
+        self.lifetime = lifetime
+        self.no_datastore = no_datastore
+        self.cookie_only_thresh = cookie_only_threshold
+        self.base_key = cookie_key
+
+        if sid:
+            self.__set_sid(sid, False)
+            self.data = None
+        else:
+            self.__read_cookie()
+
+    @staticmethod
+    def __compute_hmac(base_key, sid, text):
+        """Computes the signature for text given base_key and sid."""
+        key = base_key + sid
+        return b64encode(hmac.new(key, text, hashlib.sha256).digest())
+
+    def __read_cookie(self):
+        """Reads the HTTP Cookie and loads the sid and data from it (if any)."""
+        try:
+            # check the cookie to see if a session has been started
+            cookie = SimpleCookie(os.environ['HTTP_COOKIE'])
+            self.cookie_keys = filter(is_gaesessions_key, cookie.keys())
+            if not self.cookie_keys:
+                return  # no session yet
+            self.cookie_keys.sort()
+            data = ''.join(cookie[k].value for k in self.cookie_keys)
+            i = SIG_LEN + SID_LEN
+            sig, sid, b64pdump = data[:SIG_LEN], data[SIG_LEN:i], data[i:]
+            pdump = b64decode(b64pdump)
+            actual_sig = Session.__compute_hmac(self.base_key, sid, pdump)
+            if sig == actual_sig:
+                self.__set_sid(sid, False)
+                # check for expiration and terminate the session if it has expired
+                if time.time() > self.get_expiration():
+                    return self.terminate()
+
+                if pdump:
+                    self.data = self.__decode_data(pdump)
+                else:
+                    self.data = None  # data is in memcache/db: load it on-demand
+            else:
+                logging.warn('cookie with invalid sig received from %s: %s' % (os.environ.get('REMOTE_ADDR'), b64pdump))
+        except (CookieError, KeyError, IndexError, TypeError):
+            # there is no cookie (i.e., no session) or the cookie is invalid
+            self.terminate(False)
+
+    def make_cookie_headers(self):
+        """Returns a list of cookie headers to send (if any)."""
+        # expire all cookies if the session has ended
+        if not self.sid:
+            return [EXPIRE_COOKIE_FMT % k for k in self.cookie_keys]
+
+        if self.cookie_data is None:
+            return []  # no cookie headers need to be sent
+
+        # build the cookie header(s): includes sig, sid, and cookie_data
+        if self.is_ssl_only():
+            m = MAX_DATA_PER_COOKIE - 8
+            fmt = COOKIE_FMT_SECURE
+        else:
+            m = MAX_DATA_PER_COOKIE
+            fmt = COOKIE_FMT
+        sig = Session.__compute_hmac(self.base_key, self.sid, self.cookie_data)
+        cv = sig + self.sid + b64encode(self.cookie_data)
+        num_cookies = 1 + (len(cv) - 1) / m
+        ed = datetime.datetime.fromtimestamp(self.get_expiration()).strftime(COOKIE_DATE_FMT)
+        cookies = [fmt % (i, cv[i*m:i*m+m], ed) for i in xrange(num_cookies)]
+
+        # expire old cookies which aren't needed anymore
+        old_cookies = xrange(num_cookies, len(self.cookie_keys))
+        key = COOKIE_NAME_PREFIX + '%02d'
+        cookies_to_ax = [EXPIRE_COOKIE_FMT % (key % i) for i in old_cookies]
+        return cookies + cookies_to_ax
+
+    def is_active(self):
+        """Returns True if this session is active (i.e., it has been assigned a
+        session ID and will be or has been persisted)."""
+        return self.sid is not None
+
+    def is_ssl_only(self):
+        """Returns True if cookies set by this session will include the "Secure"
+        attribute so that the client will only send them over a secure channel
+        like SSL)."""
+        return self.sid is not None and self.sid[-33]=='S'
+
+    def ensure_data_loaded(self):
+        """Fetch the session data if it hasn't been retrieved it yet."""
+        if self.data is None and self.sid:
+            self.__retrieve_data()
+
+    def get_expiration(self):
+        """Returns the timestamp at which this session will expire."""
+        try:
+            return int(self.sid[:-33])
+        except:
+            return 0
+
+    def __make_sid(self, expire_ts=None, ssl_only=False):
+        """Returns a new session ID."""
+        # make a random ID (random.randrange() is 10x faster but less secure?)
+        if not expire_ts:
+            expire_dt = datetime.datetime.now() + self.lifetime
+            expire_ts = int(time.mktime((expire_dt).timetuple()))
+        else:
+            expire_ts = int(expire_ts)
+        if ssl_only:
+            sep = 'S'
+        else:
+            sep = '_'
+        return str(expire_ts) + sep + hashlib.md5(os.urandom(16)).hexdigest()
+
+    @staticmethod
+    def __encode_data(d):
+        """Returns a "pickled+" encoding of d.  d values of type db.Model are
+        protobuf encoded before pickling to minimize CPU usage & data size."""
+        # separate protobufs so we'll know how to decode (they are just strings)
+        eP = {} # for models encoded as protobufs
+        eO = {} # for everything else
+        for k,v in d.iteritems():
+            if isinstance(v, db.Model):
+                eP[k] = db.model_to_protobuf(v)
+            else:
+                eO[k] = v
+        return pickle.dumps((eP,eO), 2)
+
+    @staticmethod
+    def __decode_data(pdump):
+        """Returns a data dictionary after decoding it from "pickled+" form."""
+        eP, eO = pickle.loads(pdump)
+        for k,v in eP.iteritems():
+            eO[k] = db.model_from_protobuf(v)
+        return eO
+
+    def regenerate_id(self, expiration_ts=None):
+        """Assigns the session a new session ID (data carries over).  This
+        should be called whenever a user authenticates to prevent session
+        fixation attacks.
+
+        ``expiration_ts`` - The UNIX timestamp the session will expire at. If
+        omitted, the session expiration time will not be changed.
+        """
+        if self.sid:
+            self.ensure_data_loaded()  # ensure we have the data before we delete it
+            if expiration_ts is None:
+                expiration_ts = self.get_expiration()
+            self.__set_sid(self.__make_sid(expiration_ts, self.is_ssl_only()))
+            self.dirty = True  # ensure the data is written to the new session
+
+    def start(self, expiration_ts=None, ssl_only=False):
+        """Starts a new session.  expiration specifies when it will expire.  If
+        expiration is not specified, then self.lifetime will used to
+        determine the expiration date.
+
+        Normally this method does not need to be called directly - a session is
+        automatically started when the first value is added to the session.
+
+        ``expiration_ts`` - The UNIX timestamp the session will expire at. If
+        omitted, the session will expire after the default ``lifetime`` has past
+        (as specified in ``SessionMiddleware``).
+
+        ``ssl_only`` - Whether to specify the "Secure" attribute on the cookie
+        so that the client will ONLY transfer the cookie over a secure channel.
+        """
+        self.dirty = True
+        self.data = {}
+        self.__set_sid(self.__make_sid(expiration_ts, ssl_only), True)
+
+    def terminate(self, clear_data=True):
+        """Deletes the session and its data, and expires the user's cookie."""
+        if clear_data:
+            self.__clear_data()
+        self.sid = None
+        self.data = {}
+        self.dirty = False
+        if self.cookie_keys:
+            self.cookie_data = ''  # trigger the cookies to expire
+        else:
+            self.cookie_data = None
+
+    def __set_sid(self, sid, make_cookie=True):
+        """Sets the session ID, deleting the old session if one existed.  The
+        session's data will remain intact (only the session ID changes)."""
+        if self.sid:
+            self.__clear_data()
+        self.sid = sid
+        self.db_key = db.Key.from_path(SessionModel.kind(), sid, namespace='')
+
+        # set the cookie if requested
+        if make_cookie:
+            self.cookie_data = ''  # trigger the cookie to be sent
+
+    def __clear_data(self):
+        """Deletes this session from memcache and the datastore."""
+        if self.sid:
+            memcache.delete(self.sid, namespace='') # not really needed; it'll go away on its own
+            try:
+                db.delete(self.db_key)
+            except:
+                pass # either it wasn't in the db (maybe cookie/memcache-only) or db is down => cron will expire it
+
+    def __retrieve_data(self):
+        """Sets the data associated with this session after retrieving it from
+        memcache or the datastore.  Assumes self.sid is set.  Checks for session
+        expiration after getting the data."""
+        pdump = memcache.get(self.sid, namespace='')
+        if pdump is None:
+            # memcache lost it, go to the datastore
+            if self.no_datastore:
+                logging.info("can't find session data in memcache for sid=%s (using memcache only sessions)" % self.sid)
+                self.terminate(False) # we lost it; just kill the session
+                return
+            session_model_instance = db.get(self.db_key)
+            if session_model_instance:
+                pdump = session_model_instance.pdump
+            else:
+                logging.error("can't find session data in the datastore for sid=%s" % self.sid)
+                self.terminate(False) # we lost it; just kill the session
+                return
+        self.data = self.__decode_data(pdump)
+
+    def save(self, persist_even_if_using_cookie=False):
+        """Saves the data associated with this session IF any changes have been
+        made (specifically, if any mutator methods like __setitem__ or the like
+        is called).
+
+        If the data is small enough it will be sent back to the user in a cookie
+        instead of using memcache and the datastore.  If `persist_even_if_using_cookie`
+        evaluates to True, memcache and the datastore will also be used.  If the
+        no_datastore option is set, then the datastore will never be used.
+
+        Normally this method does not need to be called directly - a session is
+        automatically saved at the end of the request if any changes were made.
+        """
+        if not self.sid:
+            return # no session is active
+        if not self.dirty:
+            return # nothing has changed
+        dirty = self.dirty
+        self.dirty = False  # saving, so it won't be dirty anymore
+
+        # do the pickling ourselves b/c we need it for the datastore anyway
+        pdump = self.__encode_data(self.data)
+
+        # persist via cookies if it is reasonably small
+        if len(pdump)*4/3 <= self.cookie_only_thresh: # 4/3 b/c base64 is ~33% bigger
+            self.cookie_data = pdump
+            if not persist_even_if_using_cookie:
+                return
+        elif self.cookie_keys:
+            # latest data will only be in the backend, so expire data cookies we set
+            self.cookie_data = ''
+
+        memcache.set(self.sid, pdump, namespace='')  # may fail if memcache is down
+
+        # persist the session to the datastore
+        if dirty is Session.DIRTY_BUT_DONT_PERSIST_TO_DB or self.no_datastore:
+            return
+        try:
+            SessionModel(key_name=self.sid, pdump=pdump).put()
+        except Exception, e:
+            logging.warning("unable to persist session to datastore for sid=%s (%s)" % (self.sid,e))
+
+    # Users may interact with the session through a dictionary-like interface.
+    def clear(self):
+        """Removes all data from the session (but does not terminate it)."""
+        if self.sid:
+            self.data = {}
+            self.dirty = True
+
+    def get(self, key, default=None):
+        """Retrieves a value from the session."""
+        self.ensure_data_loaded()
+        return self.data.get(key, default)
+
+    def has_key(self, key):
+        """Returns True if key is set."""
+        self.ensure_data_loaded()
+        return self.data.has_key(key)
+
+    def pop(self, key, default=None):
+        """Removes key and returns its value, or default if key is not present."""
+        self.ensure_data_loaded()
+        self.dirty = True
+        return self.data.pop(key, default)
+
+    def pop_quick(self, key, default=None):
+        """Removes key and returns its value, or default if key is not present.
+        The change will only be persisted to memcache until another change
+        necessitates a write to the datastore."""
+        self.ensure_data_loaded()
+        if self.dirty is False:
+            self.dirty = Session.DIRTY_BUT_DONT_PERSIST_TO_DB
+        return self.data.pop(key, default)
+
+    def set_quick(self, key, value):
+        """Set a value named key on this session.  The change will only be
+        persisted to memcache until another change necessitates a write to the
+        datastore.  This will start a session if one is not already active."""
+        dirty = self.dirty
+        self[key] = value
+        if dirty is False or dirty is Session.DIRTY_BUT_DONT_PERSIST_TO_DB:
+            self.dirty = Session.DIRTY_BUT_DONT_PERSIST_TO_DB
+
+    def __getitem__(self, key):
+        """Returns the value associated with key on this session."""
+        self.ensure_data_loaded()
+        return self.data.__getitem__(key)
+
+    def __setitem__(self, key, value):
+        """Set a value named key on this session.  This will start a session if
+        one is not already active."""
+        self.ensure_data_loaded()
+        if not self.sid:
+            self.start()
+        self.data.__setitem__(key, value)
+        self.dirty = True
+
+    def __delitem__(self, key):
+        """Deletes the value associated with key on this session."""
+        self.ensure_data_loaded()
+        self.data.__delitem__(key)
+        self.dirty = True
+
+    def __iter__(self):
+        """Returns an iterator over the keys (names) of the stored values."""
+        self.ensure_data_loaded()
+        return self.data.iterkeys()
+
+    def __contains__(self, key):
+        """Returns True if key is present on this session."""
+        self.ensure_data_loaded()
+        return self.data.__contains__(key)
+
+    def __str__(self):
+        """Returns a string representation of the session."""
+        if self.sid:
+            self.ensure_data_loaded()
+            return "SID=%s %s" % (self.sid, self.data)
+        else:
+            return "uninitialized session"
+
+class SessionMiddleware(object):
+    """WSGI middleware that adds session support.
+
+    ``cookie_key`` - A key used to secure cookies so users cannot modify their
+    content.  Keys should be at least 32 bytes (RFC2104).  Tip: generate your
+    key using ``os.urandom(64)`` but do this OFFLINE and copy/paste the output
+    into a string which you pass in as ``cookie_key``.  If you use ``os.urandom()``
+    to dynamically generate your key at runtime then any existing sessions will
+    become junk every time your app starts up!
+
+    ``lifetime`` - ``datetime.timedelta`` that specifies how long a session may last.  Defaults to 7 days.
+
+    ``no_datastore`` - By default all writes also go to the datastore in case
+    memcache is lost.  Set to True to never use the datastore. This improves
+    write performance but sessions may be occassionally lost.
+
+    ``cookie_only_threshold`` - A size in bytes.  If session data is less than this
+    threshold, then session data is kept only in a secure cookie.  This avoids
+    memcache/datastore latency which is critical for small sessions.  Larger
+    sessions are kept in memcache+datastore instead.  Defaults to 10KB.
+    """
+    def __init__(self, app, cookie_key, lifetime=DEFAULT_LIFETIME, no_datastore=False, cookie_only_threshold=DEFAULT_COOKIE_ONLY_THRESH):
+        self.app = app
+        self.lifetime = lifetime
+        self.no_datastore = no_datastore
+        self.cookie_only_thresh = cookie_only_threshold
+        self.cookie_key = cookie_key
+        if not self.cookie_key:
+            raise ValueError("cookie_key MUST be specified")
+        if len(self.cookie_key) < 32:
+            raise ValueError("RFC2104 recommends you use at least a 32 character key.  Try os.urandom(64) to make a key.")
+
+    def __call__(self, environ, start_response):
+        # initialize a session for the current user
+        global _current_session
+        _current_session = Session(lifetime=self.lifetime, no_datastore=self.no_datastore, cookie_only_threshold=self.cookie_only_thresh, cookie_key=self.cookie_key)
+
+        # create a hook for us to insert a cookie into the response headers
+        def my_start_response(status, headers, exc_info=None):
+            _current_session.save() # store the session if it was changed
+            for ch in _current_session.make_cookie_headers():
+                headers.append(('Set-Cookie', ch))
+            return start_response(status, headers, exc_info)
+
+        # let the app do its thing
+        return self.app(environ, my_start_response)
+
+class DjangoSessionMiddleware(object):
+    """Django middleware that adds session support.  You must specify the
+    session configuration parameters by modifying the call to ``SessionMiddleware``
+    in ``DjangoSessionMiddleware.__init__()`` since Django cannot call an
+    initialization method with parameters.
+    """
+    def __init__(self):
+        fake_app = lambda environ, start_response : start_response
+        self.wrapped_wsgi_middleware = SessionMiddleware(fake_app, cookie_key='you MUST change this')
+        self.response_handler = None
+
+    def process_request(self, request):
+        self.response_handler = self.wrapped_wsgi_middleware(None, lambda status, headers, exc_info : headers)
+        request.session = get_current_session()  # for convenience
+
+    def process_response(self, request, response):
+        if self.response_handler:
+            session_headers = self.response_handler(None, [], None)
+            for k,v in session_headers:
+                response[k] = v
+            self.response_handler = None
+        return response
+
+def delete_expired_sessions():
+    """Deletes expired sessions from the datastore.
+    If there are more than 500 expired sessions, only 500 will be removed.
+    Returns True if all expired sessions have been removed.
+    """
+    now_str = unicode(int(time.time()))
+    q = db.Query(SessionModel, keys_only=True, namespace='')
+    key = db.Key.from_path('SessionModel', now_str + u'\ufffd', namespace='')
+    q.filter('__key__ < ', key)
+    results = q.fetch(500)
+    db.delete(results)
+    logging.info('gae-sessions: deleted %d expired sessions from the datastore' % len(results))
+    return len(results)<500
Add a comment to this file

gaesessions/__init__.pyc

Binary file added.

 
 def listPages(request):
     pages = models.Page.all()
-    return render_to_response('admin_pagelist.html', {
+    return render_to_response('admin/pagelist.html', {
                                                      'pages':pages})
 
 def newPage(request):
     if pageForm is None:
         pageForm = PageForm()
 
-    return render_to_response('admin_newpage.html', {
+    return render_to_response('admin/newpage.html', {
                                                      'pageForm':pageForm})
 
 
                                          'body':page.body,
                                          'template':page.template,
                                          'publish':page.publish})
-    return render_to_response('admin_newpage.html', {'pageForm':pageForm,
+    return render_to_response('admin/newpage.html', {'pageForm':pageForm,
                                                      'action':page.get_edit_url()})
 
 def delPage(request, key_name):
 
 def contact(request):
     form = None
+    msg = None
     if request.method == 'POST':
         newMessage = ContactForm(request.POST)
         if newMessage.is_valid():
+            data = newMessage.cleaned_data
             #send message
             mail.send_mail(sender="hudarsono.appspot.com <contact@hudarsono.appspot.com>",
                               to="Hudarsono <hudarsono@gmail.com>",
-                              subject="New Message from "+newMessage.name,
-                              body="Sender Email : "+newMessage.email+"\n Message : "+newMessage.message)
+                              subject="New Message from "+data['name'],
+                              body="Sender Email : "+data['email']+"\n Message : "+data['message'])
+
+            msg = 'Thanks for your message, will get back to you soon.'
         else:
             form = ContactForm(request.POST)
 
     if form is None: form = ContactForm()
 
-    return render_to_response('pages/contact.html', {'form':form},
+    return render_to_response('pages/contact.html', {'form':form, 'msg':msg},
                                                    context_instance=RequestContext(request))

Binary file modified.

 
 def listPost(request):
 	posts = models.Post.all()
-	return render_to_response('admin_postlist.html', {
+	return render_to_response('admin/postlist.html', {
 													  'posts':posts})
 
 
 	# get tag and categories
 	cat_tag = get_tag_cat_list()
 
-	return render_to_response('stream.html', {'posts': posts,
-											  'categories': cat_tag['cat_list'],
-											  'tags': cat_tag['tag_list']},
-                           						context_instance=RequestContext(request))
+	return render_to_response('front/stream.html', {'posts': posts,
+													  'categories': cat_tag['cat_list'],
+													  'tags': cat_tag['tag_list']},
+		                           						context_instance=RequestContext(request))
+
 
 
 def listPostByCategory(request, cat):
 	# get tag and categories
 	cat_tag = get_tag_cat_list()
 
-	return render_to_response('stream.html', {'posts': posts,
+	return render_to_response('front/stream.html', {'posts': posts,
 											  'categories': cat_tag['cat_list'],
 											  'tags': cat_tag['tag_list']},
                            						context_instance=RequestContext(request))
 
 
+
 def listPostByTag(request, tag):
 	posts = models.Post.all().filter('tags =', tag)
 
 	# get tag and categories
 	cat_tag = get_tag_cat_list()
-	return render_to_response('stream.html', {'posts': posts,
+	return render_to_response('front/stream.html', {'posts': posts,
 											  'categories': cat_tag['cat_list'],
 											  'tags': cat_tag['tag_list']},
                            						context_instance=RequestContext(request))
 
 
 def showPost(request, year, month, day, key_name):
-	post = models.Post.get_by_key_name(key_name.replace('-',' '))
+	post = models.Post.get_by_key_name(key_name)
 
 	if post:
 		# get tag and categories
 		cat_tag = get_tag_cat_list()
 
-		return render_to_response('post.html', {'post': post,
-												  'categories': cat_tag['cat_list'],
-												  'tags': cat_tag['tag_list']},
-	                           						context_instance=RequestContext(request))
+		return render_to_response('front/post.html', {'post': post,
+													  'categories': cat_tag['cat_list'],
+													  'tags': cat_tag['tag_list']},
+		                           						context_instance=RequestContext(request))
 	else:
 		raise Http404
 
+
 def newPost(request):
 	postForm = None
 	if request.method == 'POST':
 
 	if postForm is None:
 		postForm = postform.PostForm()
-	return render_to_response('admin_newpost.html', {
+	return render_to_response('admin/newpost.html', {
 													'postForm':postForm})
 
 
 												  'body': post.body,
 												  'category': post.category,
 												  'tags': ' '.join(post.tags)})
-		return render_to_response('admin_newpost.html', {
+		return render_to_response('admin/newpost.html', {
 														 'postForm':editPostForm,
 														 'action':post.get_edit_url(),})
 
+
 def delPost(request, year, month, day, key_name):
 	post = models.Post.get_by_key_name(key_name)
 	if post:

Binary file modified.

pygments/__init__.py

+# -*- coding: utf-8 -*-
+"""
+    Pygments
+    ~~~~~~~~
+
+    Pygments is a syntax highlighting package written in Python.
+
+    It is a generic syntax highlighter for general use in all kinds of software
+    such as forum systems, wikis or other applications that need to prettify
+    source code. Highlights are:
+
+    * a wide range of common languages and markup formats is supported
+    * special attention is paid to details, increasing quality by a fair amount
+    * support for new languages and formats are added easily
+    * a number of output formats, presently HTML, LaTeX, RTF, SVG, all image
+      formats that PIL supports, and ANSI sequences
+    * it is usable as a command-line tool and as a library
+    * ... and it highlights even Brainfuck!
+
+    The `Pygments tip`_ is installable with ``easy_install Pygments==dev``.
+
+    .. _Pygments tip:
+       http://bitbucket.org/birkenfeld/pygments-main/get/tip.zip#egg=Pygments-dev
+
+    :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+__version__ = '1.3.1'
+__docformat__ = 'restructuredtext'
+
+__all__ = ['lex', 'format', 'highlight']
+
+
+import sys
+
+from pygments.util import StringIO, BytesIO
+
+
+def lex(code, lexer):
+    """
+    Lex ``code`` with ``lexer`` and return an iterable of tokens.
+    """
+    try:
+        return lexer.get_tokens(code)
+    except TypeError, err:
+        if isinstance(err.args[0], str) and \
+           'unbound method get_tokens' in err.args[0]:
+            raise TypeError('lex() argument must be a lexer instance, '
+                            'not a class')
+        raise
+
+
+def format(tokens, formatter, outfile=None):
+    """
+    Format a tokenlist ``tokens`` with the formatter ``formatter``.
+
+    If ``outfile`` is given and a valid file object (an object
+    with a ``write`` method), the result will be written to it, otherwise
+    it is returned as a string.
+    """
+    try:
+        if not outfile:
+            #print formatter, 'using', formatter.encoding
+            realoutfile = formatter.encoding and BytesIO() or StringIO()
+            formatter.format(tokens, realoutfile)
+            return realoutfile.getvalue()
+        else:
+            formatter.format(tokens, outfile)
+    except TypeError, err:
+        if isinstance(err.args[0], str) and \
+           'unbound method format' in err.args[0]:
+            raise TypeError('format() argument must be a formatter instance, '
+                            'not a class')
+        raise
+
+
+def highlight(code, lexer, formatter, outfile=None):
+    """
+    Lex ``code`` with ``lexer`` and format it with the formatter ``formatter``.
+
+    If ``outfile`` is given and a valid file object (an object
+    with a ``write`` method), the result will be written to it, otherwise
+    it is returned as a string.
+    """
+    return format(lex(code, lexer), formatter, outfile)
+
+
+if __name__ == '__main__':
+    from pygments.cmdline import main
+    sys.exit(main(sys.argv))

Binary file added.

pygments/cmdline.py

+# -*- coding: utf-8 -*-
+"""
+    pygments.cmdline
+    ~~~~~~~~~~~~~~~~
+
+    Command line interface.
+
+    :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+import sys
+import getopt
+from textwrap import dedent
+
+from pygments import __version__, highlight
+from pygments.util import ClassNotFound, OptionError, docstring_headline
+from pygments.lexers import get_all_lexers, get_lexer_by_name, get_lexer_for_filename, \
+     find_lexer_class, guess_lexer, TextLexer
+from pygments.formatters import get_all_formatters, get_formatter_by_name, \
+     get_formatter_for_filename, find_formatter_class, \
+     TerminalFormatter  # pylint:disable-msg=E0611
+from pygments.filters import get_all_filters, find_filter_class
+from pygments.styles import get_all_styles, get_style_by_name
+
+
+USAGE = """\
+Usage: %s [-l <lexer> | -g] [-F <filter>[:<options>]] [-f <formatter>]
+          [-O <options>] [-P <option=value>] [-o <outfile>] [<infile>]
+
+       %s -S <style> -f <formatter> [-a <arg>] [-O <options>] [-P <option=value>]
+       %s -L [<which> ...]
+       %s -N <filename>
+       %s -H <type> <name>
+       %s -h | -V
+
+Highlight the input file and write the result to <outfile>.
+
+If no input file is given, use stdin, if -o is not given, use stdout.
+
+<lexer> is a lexer name (query all lexer names with -L). If -l is not
+given, the lexer is guessed from the extension of the input file name
+(this obviously doesn't work if the input is stdin).  If -g is passed,
+attempt to guess the lexer from the file contents, or pass through as
+plain text if this fails (this can work for stdin).
+
+Likewise, <formatter> is a formatter name, and will be guessed from
+the extension of the output file name. If no output file is given,
+the terminal formatter will be used by default.
+
+With the -O option, you can give the lexer and formatter a comma-
+separated list of options, e.g. ``-O bg=light,python=cool``.
+
+The -P option adds lexer and formatter options like the -O option, but
+you can only give one option per -P. That way, the option value may
+contain commas and equals signs, which it can't with -O, e.g.
+``-P "heading=Pygments, the Python highlighter".
+
+With the -F option, you can add filters to the token stream, you can
+give options in the same way as for -O after a colon (note: there must
+not be spaces around the colon).
+
+The -O, -P and -F options can be given multiple times.
+
+With the -S option, print out style definitions for style <style>
+for formatter <formatter>. The argument given by -a is formatter
+dependent.
+
+The -L option lists lexers, formatters, styles or filters -- set
+`which` to the thing you want to list (e.g. "styles"), or omit it to
+list everything.
+
+The -N option guesses and prints out a lexer name based solely on
+the given filename. It does not take input or highlight anything.
+If no specific lexer can be determined "text" is returned.
+
+The -H option prints detailed help for the object <name> of type <type>,
+where <type> is one of "lexer", "formatter" or "filter".
+
+The -h option prints this help.
+The -V option prints the package version.
+"""
+
+
+def _parse_options(o_strs):
+    opts = {}
+    if not o_strs:
+        return opts
+    for o_str in o_strs:
+        if not o_str:
+            continue
+        o_args = o_str.split(',')
+        for o_arg in o_args:
+            o_arg = o_arg.strip()
+            try:
+                o_key, o_val = o_arg.split('=')
+                o_key = o_key.strip()
+                o_val = o_val.strip()
+            except ValueError:
+                opts[o_arg] = True
+            else:
+                opts[o_key] = o_val
+    return opts
+
+
+def _parse_filters(f_strs):
+    filters = []
+    if not f_strs:
+        return filters
+    for f_str in f_strs:
+        if ':' in f_str:
+            fname, fopts = f_str.split(':', 1)
+            filters.append((fname, _parse_options([fopts])))
+        else:
+            filters.append((f_str, {}))
+    return filters
+
+
+def _print_help(what, name):
+    try:
+        if what == 'lexer':
+            cls = find_lexer_class(name)
+            print "Help on the %s lexer:" % cls.name
+            print dedent(cls.__doc__)
+        elif what == 'formatter':
+            cls = find_formatter_class(name)
+            print "Help on the %s formatter:" % cls.name
+            print dedent(cls.__doc__)
+        elif what == 'filter':
+            cls = find_filter_class(name)
+            print "Help on the %s filter:" % name
+            print dedent(cls.__doc__)
+    except AttributeError:
+        print >>sys.stderr, "%s not found!" % what
+
+
+def _print_list(what):
+    if what == 'lexer':
+        print
+        print "Lexers:"
+        print "~~~~~~~"
+
+        info = []
+        for fullname, names, exts, _ in get_all_lexers():
+            tup = (', '.join(names)+':', fullname,
+                   exts and '(filenames ' + ', '.join(exts) + ')' or '')
+            info.append(tup)
+        info.sort()
+        for i in info:
+            print ('* %s\n    %s %s') % i
+
+    elif what == 'formatter':
+        print
+        print "Formatters:"
+        print "~~~~~~~~~~~"
+
+        info = []
+        for cls in get_all_formatters():
+            doc = docstring_headline(cls)
+            tup = (', '.join(cls.aliases) + ':', doc, cls.filenames and
+                   '(filenames ' + ', '.join(cls.filenames) + ')' or '')
+            info.append(tup)
+        info.sort()
+        for i in info:
+            print ('* %s\n    %s %s') % i
+
+    elif what == 'filter':
+        print
+        print "Filters:"
+        print "~~~~~~~~"
+
+        for name in get_all_filters():
+            cls = find_filter_class(name)
+            print "* " + name + ':'
+            print "    %s" % docstring_headline(cls)
+
+    elif what == 'style':
+        print
+        print "Styles:"
+        print "~~~~~~~"
+
+        for name in get_all_styles():
+            cls = get_style_by_name(name)
+            print "* " + name + ':'
+            print "    %s" % docstring_headline(cls)
+
+
+def main(args=sys.argv):
+    """
+    Main command line entry point.
+    """
+    # pylint: disable-msg=R0911,R0912,R0915
+
+    usage = USAGE % ((args[0],) * 6)
+
+    try:
+        popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:hVHg")
+    except getopt.GetoptError, err:
+        print >>sys.stderr, usage
+        return 2
+    opts = {}
+    O_opts = []
+    P_opts = []
+    F_opts = []
+    for opt, arg in popts:
+        if opt == '-O':
+            O_opts.append(arg)
+        elif opt == '-P':
+            P_opts.append(arg)
+        elif opt == '-F':
+            F_opts.append(arg)
+        opts[opt] = arg
+
+    if not opts and not args:
+        print usage
+        return 0
+
+    if opts.pop('-h', None) is not None:
+        print usage
+        return 0
+
+    if opts.pop('-V', None) is not None:
+        print 'Pygments version %s, (c) 2006-2008 by Georg Brandl.' % __version__
+        return 0
+
+    # handle ``pygmentize -L``
+    L_opt = opts.pop('-L', None)
+    if L_opt is not None:
+        if opts:
+            print >>sys.stderr, usage
+            return 2
+
+        # print version
+        main(['', '-V'])
+        if not args:
+            args = ['lexer', 'formatter', 'filter', 'style']
+        for arg in args:
+            _print_list(arg.rstrip('s'))
+        return 0
+
+    # handle ``pygmentize -H``
+    H_opt = opts.pop('-H', None)
+    if H_opt is not None:
+        if opts or len(args) != 2:
+            print >>sys.stderr, usage
+            return 2
+
+        what, name = args
+        if what not in ('lexer', 'formatter', 'filter'):
+            print >>sys.stderr, usage
+            return 2
+
+        _print_help(what, name)
+        return 0
+
+    # parse -O options
+    parsed_opts = _parse_options(O_opts)
+    opts.pop('-O', None)
+
+    # parse -P options
+    for p_opt in P_opts:
+        try:
+            name, value = p_opt.split('=', 1)
+        except ValueError:
+            parsed_opts[p_opt] = True
+        else:
+            parsed_opts[name] = value
+    opts.pop('-P', None)
+
+    # handle ``pygmentize -N``
+    infn = opts.pop('-N', None)
+    if infn is not None:
+        try:
+            lexer = get_lexer_for_filename(infn, **parsed_opts)
+        except ClassNotFound, err:
+            lexer = TextLexer()
+        except OptionError, err:
+            print >>sys.stderr, 'Error:', err
+            return 1
+
+        print lexer.aliases[0]
+        return 0
+
+    # handle ``pygmentize -S``
+    S_opt = opts.pop('-S', None)
+    a_opt = opts.pop('-a', None)
+    if S_opt is not None:
+        f_opt = opts.pop('-f', None)
+        if not f_opt:
+            print >>sys.stderr, usage
+            return 2
+        if opts or args:
+            print >>sys.stderr, usage
+            return 2
+
+        try:
+            parsed_opts['style'] = S_opt
+            fmter = get_formatter_by_name(f_opt, **parsed_opts)
+        except ClassNotFound, err:
+            print >>sys.stderr, err
+            return 1
+
+        arg = a_opt or ''
+        try:
+            print fmter.get_style_defs(arg)
+        except Exception, err:
+            print >>sys.stderr, 'Error:', err
+            return 1
+        return 0
+
+    # if no -S is given, -a is not allowed
+    if a_opt is not None:
+        print >>sys.stderr, usage
+        return 2
+
+    # parse -F options
+    F_opts = _parse_filters(F_opts)
+    opts.pop('-F', None)
+
+    # select formatter
+    outfn = opts.pop('-o', None)
+    fmter = opts.pop('-f', None)
+    if fmter:
+        try:
+            fmter = get_formatter_by_name(fmter, **parsed_opts)
+        except (OptionError, ClassNotFound), err:
+            print >>sys.stderr, 'Error:', err
+            return 1
+
+    if outfn:
+        if not fmter:
+            try:
+                fmter = get_formatter_for_filename(outfn, **parsed_opts)
+            except (OptionError, ClassNotFound), err:
+                print >>sys.stderr, 'Error:', err
+                return 1
+        try:
+            outfile = open(outfn, 'wb')
+        except Exception, err:
+            print >>sys.stderr, 'Error: cannot open outfile:', err
+            return 1
+    else:
+        if not fmter:
+            fmter = TerminalFormatter(**parsed_opts)
+        outfile = sys.stdout
+
+    # select lexer
+    lexer = opts.pop('-l', None)
+    if lexer:
+        try:
+            lexer = get_lexer_by_name(lexer, **parsed_opts)
+        except (OptionError, ClassNotFound), err:
+            print >>sys.stderr, 'Error:', err
+            return 1
+
+    if args:
+        if len(args) > 1:
+            print >>sys.stderr, usage
+            return 2
+
+        infn = args[0]
+        try:
+            code = open(infn, 'rb').read()
+        except Exception, err:
+            print >>sys.stderr, 'Error: cannot read infile:', err
+            return 1
+
+        if not lexer:
+            try:
+                lexer = get_lexer_for_filename(infn, code, **parsed_opts)
+            except ClassNotFound, err:
+                if '-g' in opts:
+                    try:
+                        lexer = guess_lexer(code)
+                    except ClassNotFound:
+                        lexer = TextLexer()
+                else:
+                    print >>sys.stderr, 'Error:', err
+                    return 1
+            except OptionError, err:
+                print >>sys.stderr, 'Error:', err
+                return 1
+
+    else:
+        if '-g' in opts:
+            code = sys.stdin.read()
+            try:
+                lexer = guess_lexer(code)
+            except ClassNotFound:
+                lexer = TextLexer()
+        elif not lexer:
+            print >>sys.stderr, 'Error: no lexer name given and reading ' + \
+                                'from stdin (try using -g or -l <lexer>)'
+            return 2
+        else:
+            code = sys.stdin.read()
+
+    # No encoding given? Use latin1 if output file given,
+    # stdin/stdout encoding otherwise.
+    # (This is a compromise, I'm not too happy with it...)
+    if 'encoding' not in parsed_opts and 'outencoding' not in parsed_opts:
+        if outfn:
+            # encoding pass-through
+            fmter.encoding = 'latin1'
+        else:
+            if sys.version_info < (3,):
+                # use terminal encoding; Python 3's terminals already do that
+                lexer.encoding = getattr(sys.stdin, 'encoding',
+                                         None) or 'ascii'
+                fmter.encoding = getattr(sys.stdout, 'encoding',
+                                         None) or 'ascii'
+
+    # ... and do it!
+    try:
+        # process filters
+        for fname, fopts in F_opts:
+            lexer.add_filter(fname, **fopts)
+        highlight(code, lexer, fmter, outfile)
+    except Exception, err:
+        import traceback
+        info = traceback.format_exception(*sys.exc_info())
+        msg = info[-1].strip()
+        if len(info) >= 3:
+            # extract relevant file and position info
+            msg += '\n   (f%s)' % info[-2].split('\n')[0].strip()[1:]
+        print >>sys.stderr
+        print >>sys.stderr, '*** Error while highlighting:'
+        print >>sys.stderr, msg
+        return 1
+
+    return 0

pygments/console.py

+# -*- coding: utf-8 -*-
+"""
+    pygments.console
+    ~~~~~~~~~~~~~~~~
+
+    Format colored console output.
+
+    :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+esc = "\x1b["
+
+codes = {}
+codes[""]          = ""
+codes["reset"]     = esc + "39;49;00m"
+
+codes["bold"]      = esc + "01m"
+codes["faint"]     = esc + "02m"
+codes["standout"]  = esc + "03m"
+codes["underline"] = esc + "04m"
+codes["blink"]     = esc + "05m"
+codes["overline"]  = esc + "06m"
+
+dark_colors  = ["black", "darkred", "darkgreen", "brown", "darkblue",
+                "purple", "teal", "lightgray"]
+light_colors = ["darkgray", "red", "green", "yellow", "blue",
+                "fuchsia", "turquoise", "white"]
+
+x = 30
+for d, l in zip(dark_colors, light_colors):
+    codes[d] = esc + "%im" % x
+    codes[l] = esc + "%i;01m" % x
+    x += 1
+
+del d, l, x
+
+codes["darkteal"]   = codes["turquoise"]
+codes["darkyellow"] = codes["brown"]
+codes["fuscia"]     = codes["fuchsia"]
+codes["white"]      = codes["bold"]
+
+
+def reset_color():
+    return codes["reset"]
+
+
+def colorize(color_key, text):
+    return codes[color_key] + text + codes["reset"]
+
+
+def ansiformat(attr, text):
+    """
+    Format ``text`` with a color and/or some attributes::
+
+        color       normal color
+        *color*     bold color
+        _color_     underlined color
+        +color+     blinking color
+    """
+    result = []
+    if attr[:1] == attr[-1:] == '+':
+        result.append(codes['blink'])
+        attr = attr[1:-1]
+    if attr[:1] == attr[-1:] == '*':
+        result.append(codes['bold'])
+        attr = attr[1:-1]
+    if attr[:1] == attr[-1:] == '_':
+        result.append(codes['underline'])
+        attr = attr[1:-1]
+    result.append(codes[attr])
+    result.append(text)
+    result.append(codes['reset'])
+    return ''.join(result)

pygments/filter.py

+# -*- coding: utf-8 -*-
+"""
+    pygments.filter
+    ~~~~~~~~~~~~~~~
+
+    Module that implements the default filter.
+
+    :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+
+def apply_filters(stream, filters, lexer=None):
+    """
+    Use this method to apply an iterable of filters to
+    a stream. If lexer is given it's forwarded to the
+    filter, otherwise the filter receives `None`.
+    """
+    def _apply(filter_, stream):
+        for token in filter_.filter(lexer, stream):
+            yield token
+    for filter_ in filters:
+        stream = _apply(filter_, stream)
+    return stream
+
+
+def simplefilter(f):
+    """
+    Decorator that converts a function into a filter::
+
+        @simplefilter
+        def lowercase(lexer, stream, options):
+            for ttype, value in stream:
+                yield ttype, value.lower()
+    """
+    return type(f.__name__, (FunctionFilter,), {
+                'function':     f,
+                '__module__':   getattr(f, '__module__'),
+                '__doc__':      f.__doc__
+            })
+
+
+class Filter(object):
+    """
+    Default filter. Subclass this class or use the `simplefilter`
+    decorator to create own filters.
+    """
+
+    def __init__(self, **options):
+        self.options = options
+
+    def filter(self, lexer, stream):
+        raise NotImplementedError()
+
+
+class FunctionFilter(Filter):
+    """
+    Abstract class used by `simplefilter` to create simple
+    function filters on the fly. The `simplefilter` decorator
+    automatically creates subclasses of this class for
+    functions passed to it.
+    """
+    function = None
+
+    def __init__(self, **options):
+        if not hasattr(self, 'function'):
+            raise TypeError('%r used without bound function' %
+                            self.__class__.__name__)
+        Filter.__init__(self, **options)
+
+    def filter(self, lexer, stream):
+        # pylint: disable-msg=E1102
+        for ttype, value in self.function(lexer, stream, self.options):
+            yield ttype, value

pygments/filters/__init__.py

+# -*- coding: utf-8 -*-
+"""
+    pygments.filters
+    ~~~~~~~~~~~~~~~~
+
+    Module containing filter lookup functions and default
+    filters.
+
+    :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.token import String, Comment, Keyword, Name, Error, Whitespace, \
+    string_to_tokentype
+from pygments.filter import Filter
+from pygments.util import get_list_opt, get_int_opt, get_bool_opt, \
+     get_choice_opt, ClassNotFound, OptionError
+from pygments.plugin import find_plugin_filters
+
+
+def find_filter_class(filtername):
+    """
+    Lookup a filter by name. Return None if not found.
+    """
+    if filtername in FILTERS:
+        return FILTERS[filtername]
+    for name, cls in find_plugin_filters():
+        if name == filtername:
+            return cls
+    return None
+
+
+def get_filter_by_name(filtername, **options):
+    """
+    Return an instantiated filter. Options are passed to the filter
+    initializer if wanted. Raise a ClassNotFound if not found.
+    """
+    cls = find_filter_class(filtername)
+    if cls:
+        return cls(**options)
+    else:
+        raise ClassNotFound('filter %r not found' % filtername)
+
+
+def get_all_filters():
+    """
+    Return a generator of all filter names.
+    """
+    for name in FILTERS:
+        yield name
+    for name, _ in find_plugin_filters():
+        yield name
+
+
+def _replace_special(ttype, value, regex, specialttype,
+                     replacefunc=lambda x: x):
+    last = 0
+    for match in regex.finditer(value):
+        start, end = match.start(), match.end()
+        if start != last:
+            yield ttype, value[last:start]
+        yield specialttype, replacefunc(value[start:end])
+        last = end
+    if last != len(value):
+        yield ttype, value[last:]
+
+
+class CodeTagFilter(Filter):
+    """
+    Highlight special code tags in comments and docstrings.
+
+    Options accepted:
+
+    `codetags` : list of strings
+       A list of strings that are flagged as code tags.  The default is to
+       highlight ``XXX``, ``TODO``, ``BUG`` and ``NOTE``.
+    """
+
+    def __init__(self, **options):
+        Filter.__init__(self, **options)
+        tags = get_list_opt(options, 'codetags',
+                            ['XXX', 'TODO', 'BUG', 'NOTE'])
+        self.tag_re = re.compile(r'\b(%s)\b' % '|'.join([
+            re.escape(tag) for tag in tags if tag
+        ]))
+
+    def filter(self, lexer, stream):
+        regex = self.tag_re
+        for ttype, value in stream:
+            if ttype in String.Doc or \
+               ttype in Comment and \
+               ttype not in Comment.Preproc:
+                for sttype, svalue in _replace_special(ttype, value, regex,
+                                                       Comment.Special):
+                    yield sttype, svalue
+            else:
+                yield ttype, value
+
+
+class KeywordCaseFilter(Filter):
+    """
+    Convert keywords to lowercase or uppercase or capitalize them, which
+    means first letter uppercase, rest lowercase.
+
+    This can be useful e.g. if you highlight Pascal code and want to adapt the
+    code to your styleguide.
+
+    Options accepted:
+
+    `case` : string
+       The casing to convert keywords to. Must be one of ``'lower'``,
+       ``'upper'`` or ``'capitalize'``.  The default is ``'lower'``.
+    """
+
+    def __init__(self, **options):
+        Filter.__init__(self, **options)
+        case = get_choice_opt(options, 'case', ['lower', 'upper', 'capitalize'], 'lower')
+        self.convert = getattr(unicode, case)
+
+    def filter(self, lexer, stream):
+        for ttype, value in stream:
+            if ttype in Keyword:
+                yield ttype, self.convert(value)
+            else:
+                yield ttype, value
+
+
+class NameHighlightFilter(Filter):
+    """
+    Highlight a normal Name token with a different token type.
+
+    Example::
+
+        filter = NameHighlightFilter(
+            names=['foo', 'bar', 'baz'],
+            tokentype=Name.Function,
+        )
+
+    This would highlight the names "foo", "bar" and "baz"
+    as functions. `Name.Function` is the default token type.
+
+    Options accepted:
+
+    `names` : list of strings
+      A list of names that should be given the different token type.
+      There is no default.
+    `tokentype` : TokenType or string
+      A token type or a string containing a token type name that is
+      used for highlighting the strings in `names`.  The default is
+      `Name.Function`.
+    """
+
+    def __init__(self, **options):
+        Filter.__init__(self, **options)
+        self.names = set(get_list_opt(options, 'names', []))
+        tokentype = options.get('tokentype')
+        if tokentype:
+            self.tokentype = string_to_tokentype(tokentype)
+        else:
+            self.tokentype = Name.Function
+
+    def filter(self, lexer, stream):
+        for ttype, value in stream:
+            if ttype is Name and value in self.names:
+                yield self.tokentype, value
+            else:
+                yield ttype, value
+
+
+class ErrorToken(Exception):
+    pass
+
+class RaiseOnErrorTokenFilter(Filter):
+    """
+    Raise an exception when the lexer generates an error token.
+
+    Options accepted:
+
+    `excclass` : Exception class
+      The exception class to raise.
+      The default is `pygments.filters.ErrorToken`.
+
+    *New in Pygments 0.8.*
+    """
+
+    def __init__(self, **options):
+        Filter.__init__(self, **options)
+        self.exception = options.get('excclass', ErrorToken)
+        try:
+            # issubclass() will raise TypeError if first argument is not a class
+            if not issubclass(self.exception, Exception):
+                raise TypeError
+        except TypeError:
+            raise OptionError('excclass option is not an exception class')
+
+    def filter(self, lexer, stream):
+        for ttype, value in stream:
+            if ttype is Error:
+                raise self.exception(value)
+            yield ttype, value
+
+
+class VisibleWhitespaceFilter(Filter):
+    """
+    Convert tabs, newlines and/or spaces to visible characters.
+
+    Options accepted:
+
+    `spaces` : string or bool
+      If this is a one-character string, spaces will be replaces by this string.
+      If it is another true value, spaces will be replaced by ``·`` (unicode
+      MIDDLE DOT).  If it is a false value, spaces will not be replaced.  The
+      default is ``False``.
+    `tabs` : string or bool
+      The same as for `spaces`, but the default replacement character is ``»``
+      (unicode RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK).  The default value
+      is ``False``.  Note: this will not work if the `tabsize` option for the
+      lexer is nonzero, as tabs will already have been expanded then.
+    `tabsize` : int
+      If tabs are to be replaced by this filter (see the `tabs` option), this
+      is the total number of characters that a tab should be expanded to.
+      The default is ``8``.
+    `newlines` : string or bool
+      The same as for `spaces`, but the default replacement character is ``¶``
+      (unicode PILCROW SIGN).  The default value is ``False``.
+    `wstokentype` : bool
+      If true, give whitespace the special `Whitespace` token type.  This allows
+      styling the visible whitespace differently (e.g. greyed out), but it can
+      disrupt background colors.  The default is ``True``.
+
+    *New in Pygments 0.8.*
+    """
+
+    def __init__(self, **options):
+        Filter.__init__(self, **options)
+        for name, default in {'spaces': u'·', 'tabs': u'»', 'newlines': u'¶'}.items():
+            opt = options.get(name, False)
+            if isinstance(opt, basestring) and len(opt) == 1:
+                setattr(self, name, opt)
+            else:
+                setattr(self, name, (opt and default or ''))
+        tabsize = get_int_opt(options, 'tabsize', 8)
+        if self.tabs:
+            self.tabs += ' '*(tabsize-1)
+        if self.newlines:
+            self.newlines += '\n'
+        self.wstt = get_bool_opt(options, 'wstokentype', True)
+
+    def filter(self, lexer, stream):
+        if self.wstt:
+            spaces = self.spaces or ' '
+            tabs = self.tabs or '\t'
+            newlines = self.newlines or '\n'
+            regex = re.compile(r'\s')
+            def replacefunc(wschar):
+                if wschar == ' ':
+                    return spaces
+                elif wschar == '\t':
+                    return tabs
+                elif wschar == '\n':
+                    return newlines
+                return wschar
+
+            for ttype, value in stream:
+                for sttype, svalue in _replace_special(ttype, value, regex,
+                                                       Whitespace, replacefunc):
+                    yield sttype, svalue
+        else:
+            spaces, tabs, newlines = self.spaces, self.tabs, self.newlines
+            # simpler processing
+            for ttype, value in stream:
+                if spaces:
+                    value = value.replace(' ', spaces)
+                if tabs:
+                    value = value.replace('\t', tabs)
+                if newlines:
+                    value = value.replace('\n', newlines)
+                yield ttype, value
+
+
+class GobbleFilter(Filter):
+    """
+    Gobbles source code lines (eats initial characters).
+
+    This filter drops the first ``n`` characters off every line of code.  This
+    may be useful when the source code fed to the lexer is indented by a fixed
+    amount of space that isn't desired in the output.
+
+    Options accepted:
+
+    `n` : int
+       The number of characters to gobble.
+
+    *New in Pygments 1.2.*
+    """
+    def __init__(self, **options):
+        Filter.__init__(self, **options)
+        self.n = get_int_opt(options, 'n', 0)
+
+    def gobble(self, value, left):
+        if left < len(value):
+            return value[left:], 0
+        else:
+            return '', left - len(value)
+
+    def filter(self, lexer, stream):
+        n = self.n
+        left = n # How many characters left to gobble.
+        for ttype, value in stream:
+            # Remove ``left`` tokens from first line, ``n`` from all others.
+            parts = value.split('\n')
+            (parts[0], left) = self.gobble(parts[0], left)
+            for i in range(1, len(parts)):
+                (parts[i], left) = self.gobble(parts[i], n)
+            value = '\n'.join(parts)
+
+            if value != '':
+                yield ttype, value
+
+
+class TokenMergeFilter(Filter):
+    """
+    Merges consecutive tokens with the same token type in the output stream of a
+    lexer.
+
+    *New in Pygments 1.2.*
+    """
+    def __init__(self, **options):
+        Filter.__init__(self, **options)
+
+    def filter(self, lexer, stream):
+        output = []
+        current_type = None
+        current_value = None
+        for ttype, value in stream:
+            if ttype is current_type:
+                current_value += value
+            else:
+                if current_type is not None:
+                    yield current_type, current_value
+                current_type = ttype
+                current_value = value
+        if current_type is not None:
+            yield current_type, current_value
+
+
+FILTERS = {
+    'codetagify':     CodeTagFilter,
+    'keywordcase':    KeywordCaseFilter,
+    'highlight':      NameHighlightFilter,
+    'raiseonerror':   RaiseOnErrorTokenFilter,
+    'whitespace':     VisibleWhitespaceFilter,
+    'gobble':         GobbleFilter,
+    'tokenmerge':     TokenMergeFilter,
+}

pygments/formatter.py

+# -*- coding: utf-8 -*-
+"""
+    pygments.formatter
+    ~~~~~~~~~~~~~~~~~~
+
+    Base formatter class.
+
+    :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import codecs
+
+from pygments.util import get_bool_opt
+from pygments.styles import get_style_by_name
+
+__all__ = ['Formatter']
+
+
+def _lookup_style(style):
+    if isinstance(style, basestring):
+        return get_style_by_name(style)
+    return style
+
+
+class Formatter(object):
+    """
+    Converts a token stream to text.
+
+    Options accepted:
+
+    ``style``
+        The style to use, can be a string or a Style subclass
+        (default: "default"). Not used by e.g. the
+        TerminalFormatter.
+    ``full``
+        Tells the formatter to output a "full" document, i.e.
+        a complete self-contained document. This doesn't have
+        any effect for some formatters (default: false).
+    ``title``
+        If ``full`` is true, the title that should be used to
+        caption the document (default: '').
+    ``encoding``
+        If given, must be an encoding name. This will be used to
+        convert the Unicode token strings to byte strings in the
+        output. If it is "" or None, Unicode strings will be written
+        to the output file, which most file-like objects do not
+        support (default: None).
+    ``outencoding``
+        Overrides ``encoding`` if given.
+    """
+
+    #: Name of the formatter
+    name = None
+
+    #: Shortcuts for the formatter
+    aliases = []
+
+    #: fn match rules
+    filenames = []
+
+    #: If True, this formatter outputs Unicode strings when no encoding
+    #: option is given.
+    unicodeoutput = True
+
+    def __init__(self, **options):
+        self.style = _lookup_style(options.get('style', 'default'))
+        self.full  = get_bool_opt(options, 'full', False)
+        self.title = options.get('title', '')
+        self.encoding = options.get('encoding', None) or None
+        self.encoding = options.get('outencoding', None) or self.encoding
+        self.options = options
+
+    def get_style_defs(self, arg=''):
+        """
+        Return the style definitions for the current style as a string.
+
+        ``arg`` is an additional argument whose meaning depends on the
+        formatter used. Note that ``arg`` can also be a list or tuple
+        for some formatters like the html formatter.
+        """
+        return ''
+
+    def format(self, tokensource, outfile):
+        """
+        Format ``tokensource``, an iterable of ``(tokentype, tokenstring)``
+        tuples and write it into ``outfile``.
+        """
+        if self.encoding:
+            # wrap the outfile in a StreamWriter
+            outfile = codecs.lookup(self.encoding)[3](outfile)
+        return self.format_unencoded(tokensource, outfile)

pygments/formatters/__init__.py

+# -*- coding: utf-8 -*-
+"""
+    pygments.formatters
+    ~~~~~~~~~~~~~~~~~~~
+
+    Pygments formatters.
+
+    :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+import os.path
+import fnmatch
+
+from pygments.formatters._mapping import FORMATTERS
+from pygments.plugin import find_plugin_formatters
+from pygments.util import ClassNotFound
+
+ns = globals()
+for fcls in FORMATTERS:
+    ns[fcls.__name__] = fcls
+del fcls
+
+__all__ = ['get_formatter_by_name', 'get_formatter_for_filename',
+           'get_all_formatters'] + [cls.__name__ for cls in FORMATTERS]
+
+
+_formatter_alias_cache = {}
+_formatter_filename_cache = []
+
+def _init_formatter_cache():
+    if _formatter_alias_cache:
+        return
+    for cls in get_all_formatters():
+        for alias in cls.aliases:
+            _formatter_alias_cache[alias] = cls
+        for fn in cls.filenames:
+            _formatter_filename_cache.append((fn, cls))
+
+
+def find_formatter_class(name):
+    _init_formatter_cache()
+    cls = _formatter_alias_cache.get(name, None)
+    return cls
+
+
+def get_formatter_by_name(name, **options):
+    _init_formatter_cache()
+    cls = _formatter_alias_cache.get(name, None)
+    if not cls:
+        raise ClassNotFound("No formatter found for name %r" % name)
+    return cls(**options)
+
+
+def get_formatter_for_filename(fn, **options):
+    _init_formatter_cache()
+    fn = os.path.basename(fn)
+    for pattern, cls in _formatter_filename_cache:
+        if fnmatch.fnmatch(fn, pattern):
+            return cls(**options)
+    raise ClassNotFound("No formatter found for file name %r" % fn)
+
+
+def get_all_formatters():
+    """Return a generator for all formatters."""
+    for formatter in FORMATTERS:
+        yield formatter
+    for _, formatter in find_plugin_formatters():
+        yield formatter

pygments/formatters/_mapping.py

+# -*- coding: utf-8 -*-
+"""
+    pygments.formatters._mapping
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Formatter mapping defintions. This file is generated by itself. Everytime
+    you change something on a builtin formatter defintion, run this script from
+    the formatters folder to update it.
+
+    Do not alter the FORMATTERS dictionary by hand.
+
+    :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.util import docstring_headline
+
+# start
+from pygments.formatters.bbcode import BBCodeFormatter
+from pygments.formatters.html import HtmlFormatter
+from pygments.formatters.img import BmpImageFormatter
+from pygments.formatters.img import GifImageFormatter
+from pygments.formatters.img import ImageFormatter
+from pygments.formatters.img import JpgImageFormatter
+from pygments.formatters.latex import LatexFormatter
+from pygments.formatters.other import NullFormatter
+from pygments.formatters.other import RawTokenFormatter
+from pygments.formatters.rtf import RtfFormatter
+from pygments.formatters.svg import SvgFormatter
+from pygments.formatters.terminal import TerminalFormatter
+from pygments.formatters.terminal256 import Terminal256Formatter
+
+FORMATTERS = {
+    BBCodeFormatter: ('BBCode', ('bbcode', 'bb'), (), 'Format tokens with BBcodes. These formatting codes are used by many bulletin boards, so you can highlight your sourcecode with pygments before posting it there.'),
+    BmpImageFormatter: ('img_bmp', ('bmp', 'bitmap'), ('*.bmp',), 'Create a bitmap image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
+    GifImageFormatter: ('img_gif', ('gif',), ('*.gif',), 'Create a GIF image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
+    HtmlFormatter: ('HTML', ('html',), ('*.html', '*.htm'), "Format tokens as HTML 4 ``<span>`` tags within a ``<pre>`` tag, wrapped in a ``<div>`` tag. The ``<div>``'s CSS class can be set by the `cssclass` option."),
+    ImageFormatter: ('img', ('img', 'IMG', 'png'), ('*.png',), 'Create a PNG image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
+    JpgImageFormatter: ('img_jpg', ('jpg', 'jpeg'), ('*.jpg',), 'Create a JPEG image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
+    LatexFormatter: ('LaTeX', ('latex', 'tex'), ('*.tex',), 'Format tokens as LaTeX code. This needs the `fancyvrb` and `color` standard packages.'),
+    NullFormatter: ('Text only', ('text', 'null'), ('*.txt',), 'Output the text unchanged without any formatting.'),
+    RawTokenFormatter: ('Raw tokens', ('raw', 'tokens'), ('*.raw',), 'Format tokens as a raw representation for storing token streams.'),
+    RtfFormatter: ('RTF', ('rtf',), ('*.rtf',), 'Format tokens as RTF markup. This formatter automatically outputs full RTF documents with color information and other useful stuff. Perfect for Copy and Paste into Microsoft\xc2\xae Word\xc2\xae documents.'),
+    SvgFormatter: ('SVG', ('svg',), ('*.svg',), 'Format tokens as an SVG graphics file.  This formatter is still experimental. Each line of code is a ``<text>`` element with explicit ``x`` and ``y`` coordinates containing ``<tspan>`` elements with the individual token styles.'),
+    Terminal256Formatter: ('Terminal256', ('terminal256', 'console256', '256'), (), 'Format tokens with ANSI color sequences, for output in a 256-color terminal or console. Like in `TerminalFormatter` color sequences are terminated at newlines, so that paging the output works correctly.'),
+    TerminalFormatter: ('Terminal', ('terminal', 'console'), (), 'Format tokens with ANSI color sequences, for output in a text console. Color sequences are terminated at newlines, so that paging the output works correctly.')
+}
+
+if __name__ == '__main__':
+    import sys
+    import os
+
+    # lookup formatters
+    found_formatters = []
+    imports = []
+    sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
+    for filename in os.listdir('.'):
+        if filename.endswith('.py') and not filename.startswith('_'):
+            module_name = 'pygments.formatters.%s' % filename[:-3]
+            print module_name
+            module = __import__(module_name, None, None, [''])
+            for formatter_name in module.__all__:
+                imports.append((module_name, formatter_name))
+                formatter = getattr(module, formatter_name)
+                found_formatters.append(
+                    '%s: %r' % (formatter_name,
+                                (formatter.name,
+                                 tuple(formatter.aliases),
+                                 tuple(formatter.filenames),
+                                 docstring_headline(formatter))))
+    # sort them, that should make the diff files for svn smaller
+    found_formatters.sort()
+    imports.sort()
+
+    # extract useful sourcecode from this file
+    f = open(__file__)
+    try:
+        content = f.read()
+    finally:
+        f.close()
+    header = content[:content.find('# start')]
+    footer = content[content.find("if __name__ == '__main__':"):]
+
+    # write new file
+    f = open(__file__, 'w')
+    f.write(header)
+    f.write('# start\n')
+    f.write('\n'.join(['from %s import %s' % imp for imp in imports]))
+    f.write('\n\n')
+    f.write('FORMATTERS = {\n    %s\n}\n\n' % ',\n    '.join(found_formatters))
+    f.write(footer)
+    f.close()

pygments/formatters/bbcode.py

+# -*- coding: utf-8 -*-
+"""
+    pygments.formatters.bbcode
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    BBcode formatter.
+
+    :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+
+from pygments.formatter import Formatter
+from pygments.util import get_bool_opt
+
+__all__ = ['BBCodeFormatter']
+
+
+class BBCodeFormatter(Formatter):
+    """
+    Format tokens with BBcodes. These formatting codes are used by many
+    bulletin boards, so you can highlight your sourcecode with pygments before
+    posting it there.
+
+    This formatter has no support for background colors and borders, as there
+    are no common BBcode tags for that.
+
+    Some board systems (e.g. phpBB) don't support colors in their [code] tag,
+    so you can't use the highlighting together with that tag.
+    Text in a [code] tag usually is shown with a monospace font (which this
+    formatter can do with the ``monofont`` option) and no spaces (which you
+    need for indentation) are removed.
+
+    Additional options accepted:
+
+    `style`
+        The style to use, can be a string or a Style subclass (default:
+        ``'default'``).
+
+    `codetag`
+        If set to true, put the output into ``[code]`` tags (default:
+        ``false``)
+
+    `monofont`
+        If set to true, add a tag to show the code with a monospace font
+        (default: ``false``).
+    """
+    name = 'BBCode'
+    aliases = ['bbcode', 'bb']
+    filenames = []
+
+    def __init__(self, **options):
+        Formatter.__init__(self, **options)
+        self._code = get_bool_opt(options, 'codetag', False)
+        self._mono = get_bool_opt(options, 'monofont', False)
+
+        self.styles = {}
+        self._make_styles()
+
+    def _make_styles(self):
+        for ttype, ndef in self.style:
+            start = end = ''
+            if ndef['color']:
+                start += '[color=#%s]' % ndef['color']
+                end = '[/color]' + end
+            if ndef['bold']:
+                start += '[b]'
+                end = '[/b]' + end
+            if ndef['italic']:
+                start += '[i]'
+                end = '[/i]' + end
+            if ndef['underline']:
+                start += '[u]'
+                end = '[/u]' + end
+            # there are no common BBcodes for background-color and border
+
+            self.styles[ttype] = start, end
+
+    def format_unencoded(self, tokensource, outfile):
+        if self._code:
+            outfile.write('[code]')
+        if self._mono:
+            outfile.write('[font=monospace]')
+
+        lastval = ''
+        lasttype = None
+
+        for ttype, value in tokensource:
+            while ttype not in self.styles:
+                ttype = ttype.parent
+            if ttype == lasttype:
+                lastval += value
+            else:
+                if lastval:
+                    start, end = self.styles[lasttype]
+                    outfile.write(''.join((start, lastval, end)))
+                lastval = value
+                lasttype = ttype
+
+        if lastval:
+            start, end = self.styles[lasttype]
+            outfile.write(''.join((start, lastval, end)))
+
+        if self._mono:
+            outfile.write('[/font]')
+        if self._code:
+            outfile.write('[/code]')
+        if self._code or self._mono:
+            outfile.write('\n')

pygments/formatters/html.py