Source

sphinx-gsoc2009 / sphinx / builders / webapp / webapp.py

Full commit
# -*- coding: utf-8 -*-
"""
    sphinx.builders.webapp
    ~~~~~~~~~~~~~~~~~~~~~~

    A web application builder.

    :copyright: Copyright 2007-2009 by the Sphinx team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import os
import sys
import shutil
import cPickle
from os import path
from hashlib import md5

from anyvc.workdir import get_workdir_manager_for_path

import jinja2 as j2

from mercurial import commands, ui
from mercurial.hg import repository

from sphinx.builders.html import StandaloneHTMLBuilder
from sphinx.errors import SphinxError
from sphinx.web.reposums import PickleLoadSave
from sphinx.writers.html import HTMLTranslator

try:
    import xapian
except ImportError:
    xapian = None

# Xapian metadata constants
DOC_PATH = 0
DOC_TITLE = 1

piddbfile = 'piddb.pkl' # XXX: more it to a configuration file!

class WebAppBuilder(StandaloneHTMLBuilder):
    def init(self):
        self.orig_outdir = self.outdir

        # _build/webapp/public - ordinary HTML files produced by Sphinx
        self.public_dir = path.join(self.outdir, 'public')
        
        # _build/webapp/repo - reST source files as a repository
        self.repodir = path.join(self.outdir, 'repo')
        
        # _build/webapp/html - HTML templates for webapp/middleware
        self.html_dir = path.join(self.outdir, 'html')

        # init repository
        self.ui = ui.ui()

        try:
            self.copy_webapp_templates()
            # create and fill the Xapian database
            if xapian:
                self.X_db, self.X_indexer, self.X_stemmer = self.xapian_create_db()
        except Exception, err:
            raise SphinxError(err)

        self.app.add_javascript('jquery.form.js')
        self.app.add_javascript('comments.js')
        self.outdir = self.public_dir
        StandaloneHTMLBuilder.init(self)

    def get_target_uri(self, docname, typ=None):
        return docname + self.link_suffix

    def write_doc(self, docname, doctree):
        href = docname + self.out_suffix
        if xapian:
            doc_contents = doctree.astext()
            title = doc_contents[:20]
            self.xapian_fill_db(href, title, doctree.astext())
        StandaloneHTMLBuilder.write_doc(self, docname, doctree)

    def prepare_writing(self, docnames):
        StandaloneHTMLBuilder.prepare_writing(self, docnames)

    def finish(self):
        StandaloneHTMLBuilder.finish(self)
        self.outdir = self.html_dir
        self.handle_page('xapiansearch', {}, 'web/xapiansearch.html')

        self.copy_files(os.getcwd(), self.orig_outdir, (piddbfile,))
        self.create_repo()

    def init_translator_class(self):
        self.translator_class = WebappHTMLTranslator


    ###
    ### creating/detecting repositories
    ###
    def create_repo(self):
        try:
            repo = self.detect_repo()
            if repo:
                # currently - only mercurial
                commands.clone(self.ui, repo.base_path, self.repodir)
            else:
                self.new_repo()
        except Exception, err:
            raise SphinxError(err)

    def detect_repo(self):
        return get_workdir_manager_for_path(self.srcdir)

    def new_repo(self):
        commands.init(self.ui, self.repodir)
        repo = repository(self.ui, self.repodir)
        builddir = path.join(self.srcdir, '_build')
        def callback(arg, directory, files):
            srclen = len(self.srcdir)
            if directory.startswith(builddir):
                return
            for elem in files:
                srcfile = path.join(directory, elem)
                subdir = directory[srclen+1:]
                dstfile = path.join(path.join(self.repodir, subdir), elem)
                if path.isdir(srcfile):
                    os.mkdir(dstfile)
                elif not path.exists(dstfile):
                    shutil.copyfile(srcfile, dstfile)
                    commands.add(self.ui, repo, dstfile)
        path.walk(self.srcdir, callback, '')

    def copy_files(self, srcdir, dstdir, files):
        """Copy files from a list/tuple 'files' from 'srcdir' source
        directory to destination directory 'dstdir'."""
        for f in files:
            try:
                afile = path.join(srcdir, f)
                shutil.copyfile(afile, path.join(dstdir, f))
            except Exception, err:
                raise SphinxError(err)


    ###
    ### webapp templates
    ###
    def copy_webapp_templates(self):
        """
        Copy the files (defined in 'files' tuple) from sphinx/webapp/templates/
        to self.outdir.
        """
        dirs = ('html', 'comments', 'fixes')
        files = ('server.py', 'developers.txt',
                 'webapp.conf', 'html/openid.html')
        templates_dir = path.join(path.dirname(__file__), 'templates')

        for d in dirs:
            try:
                os.mkdir(path.join(self.outdir, d))
            except Exception, err:
                raise SphinxError(err)

        srcdir = path.join(path.dirname(__file__), 'templates')
        self.copy_files(srcdir, self.outdir, files)

        sphinx_dirs = (('../../themes/basic/static', 'public/_static'),
                       ('../../themes', 'themes'))
        for sdir in sphinx_dirs:
            src_dir = path.join(path.dirname(__file__), sdir[0])
            dst_dir = path.join(self.outdir, sdir[1])
            try:
                shutil.copytree(src_dir, dst_dir)
            except Exception, err:
                raise SphinxError(err)
            
    ###
    ### Xapian search engine
    ###
    def xapian_create_db(self):
        db_path = path.join(self.outdir, self.config.xapian_db)

        try:
            database = xapian.WritableDatabase(db_path, xapian.DB_CREATE_OR_OPEN)
            indexer = xapian.TermGenerator()
            stemmer = xapian.Stem("english")
            indexer.set_stemmer(stemmer)
        except Exception, err:
            raise SphinxError(err)

        return database, indexer, stemmer


    def xapian_fill_db(self, href, title, text):
        try:
            self.X_db.begin_transaction()
            doc = xapian.Document()
            doc.set_data(text)
            doc.add_value(DOC_PATH, href)
            doc.add_value(DOC_TITLE, title)
            self.X_indexer.set_document(doc)
            self.X_indexer.index_text(text)
            for word in text.split():
                doc.add_posting(word, 1)
            self.X_db.add_document(doc)
            self.X_db.commit_transaction()
        except Exception, err:
            raise SphinxError(err)

class PidDb(PickleLoadSave):
    def __init__(self, dbfile):
        self.dbfile = dbfile

    def add_record(self, filename, pid, paragraph):
        """Add a record to the database. 'filename' is key
        for the first-level dictionary, and 'pid' is a key for the
        second-level dictionary. 'paragraph' is a value assigned
        to the second-level dictionary:

        database[filename][pid] = paragraph
        """
        data = self.dbread(self.dbfile)
        if filename not in data:
            data[filename] = {}
        else:
            data[filename][pid] = paragraph
        self.dbsave(self.dbfile, data)
        

def cutcwd(fpath):
    """For path which starts with the value returned by os.getcwd() 
    return what follows. If a value to be returned starts with a slash
    ("/"), cut the slash out.
    
    For example, if the current working directory is '/etc/' and 'fpath'
    is equal to '/etc/passwd', return 'passwd'.
    
    If 'fpath' starts with something else than `os.getcwd()`, return 'fpath'."""
    cwd = os.getcwd()
    cwdlen = len(cwd)
    if fpath.startswith(cwd):
        fpath = fpath[cwdlen:]
        if fpath and fpath[0] == '/':
            return fpath[1:]
    return fpath

class WebappHTMLTranslator(HTMLTranslator):
    def __init__(self, *args, **kwds):
        self.templ_env = None # templates environment
        self.init_templates()
        self.pid = None # paragraph ID
        self.piddb = PidDb(piddbfile)
        HTMLTranslator.__init__(self, *args, **kwds)

    def init_templates(self):
        templ_path = path.join(path.dirname(__file__), 'templates/html')
        self.templ_env = j2.Environment(loader=j2.FileSystemLoader(templ_path),
                                        extensions=['jinja2.ext.i18n'])

    # overwritten (docutils.writers.html4css1:HTMLTranslator)
    def visit_paragraph(self, node):
        if self.should_be_compact_paragraph(node):
            self.context.append('')
        else:
            body = ''.join(self.body)
            self.pid = md5(body.encode('utf8')).hexdigest()
            self.body.append(self.starttag(node, 'p', NAME=self.pid))
            self.context.append('')

    def depart_paragraph(self, node):
        """Extend HTMLTranslator.depart_paragraph() by appending
        the body of the document with rendered template for comments/fixes
        post form. Also, add paragraph's rawsource (reST source) to the
        PidDb database."""
        comm_template = self.templ_env.get_template('comments.html')
        HTMLTranslator.depart_paragraph(self, node)
        if not self.should_be_compact_paragraph(node):
           self.body.append(comm_template.render({ 'paragraph_id': self.pid,
                                                   'rawsource': node.rawsource }))
           if node.source:
               self.piddb.add_record(cutcwd(node.source), self.pid, node.rawsource)