Commits

Tomasz Stachowiak committed b790037

incremental generation and upload scripts, misc changes and css for latex formulae

  • Participants
  • Parent commits 7dd75c2

Comments (0)

Files changed (5)

 *.obj
 *.map
 *.exe
+password.txt
 disqusComments = '''<div id="disqus_thread"></div><script type="text/javascript" src="http://disqus.com/forums/h3r3ticsgrimoire/embed.js"></script><noscript><a href="http://disqus.com/forums/h3r3ticsgrimoire/?url=ref">View the discussion thread.</a></noscript>'''
 
 
+class CodegenConf(object):
+	def __init__( self, dir, path, level, ver_num ):
+		self.dir = dir
+		self.path = path
+		self.level = level
+		self.outname = path[:-8]
+		self.outfile = 'output/' + dir + self.outname + '.html'
+		self.mtime = os.stat( 'input/' + dir + path ).st_mtime + ver_num
+		try:
+			self.out_of_date = os.stat( self.outfile ).st_mtime + 0.001 < self.mtime
+		except IOError:
+			self.out_of_date = True
+
+
 def formatTemplate(s, level, curPage, wantComments):
 	global disqusComments
 	return s % {
 	pipe.write(code)
 	pipe.close()
 	proc.wait()
+	os.utime( outname, (conf.mtime,) * 2 )
 	return 'p=. !%s!\n\n' % fname
 
+def generateLaTeX(code, outname, i, conf):
+	with open( 'latextemp.tex', 'w' ) as f:
+		f.write( code )
+	os.system( 'latex -quiet latextemp.tex' )
+	fname = '%s_graph%s.png' % (outname, i)
+	cmd = 'dvipng -T tight -D 190 --gamma 1.0 --truecolor -q -o %s latextemp.dvi 1>NUL' % fname
+	os.system( cmd )
+	os.system( cmd )
+	os.utime( fname, (conf.mtime,) * 2 )
+	return 'p(formula)=. !%s!\n\n' % fname
 
-def formatText(text, outname, wantComments):
+def formatText(text, outname, wantComments, conf):
 	text2 = ''
 	lines = text.splitlines()
 	graphCntr = 0
 					else: assert false, p
 				text2 += generateGraph(prefix + code + suffix, outname, graphCntr)
 				graphCntr += 1
+			elif lang[0:3] == "tex":
+				params = lang[3:].strip().split(' ')
+				prefix = r'''
+					\documentclass{article}
+					\usepackage[active]{preview}
+					\usepackage{color}
+					\begin{document}
+					\begin{preview}
+					\definecolor{bggray}{rgb}{0.215,0.215,0.215}
+					\pagecolor{bggray}
+					\setlength\fboxsep{4.0pt}
+					\setlength\fboxrule{0.2pt}
+					\color{bggray}
+					\fbox{
+					\[
+					\textcolor{white}{$
+				'''
+				suffix = r'''$}
+					\]
+					}
+					\end{preview}
+					\end{document}
+				'''
+				text2 += generateLaTeX(prefix + code + suffix, outname, graphCntr, conf)
+				graphCntr += 1
 			else:
 				lexer = get_lexer_by_name(lang, stripall=True)
 				code = highlight(code, lexer, HtmlFormatter())
 
 	return textile.textile(text2)
 
-def textile2html(dir, path, level):
-	outname = path[:-8]
-	outfile = 'output/' + dir + outname + '.html'
 
-	assert path[-8:] == '.textile'
-	inputData = open('input/'+dir+path).read()
+def textile2html( conf ):
+	outname = conf.outname
+	outfile = conf.outfile
+
+	assert conf.path[-8:] == '.textile'
+	inputData = open('input/'+conf.dir+conf.path).read()
 
 	curDir = os.getcwd()
-	os.chdir('output/'+dir)
+	os.chdir('output/'+conf.dir)
 	wantComments = [False]
-	contentsBody = formatText(inputData, outname, wantComments)
+	contentsBody = formatText(inputData, outname, wantComments, conf)
 	wantComments = wantComments[0]
 	os.chdir(curDir)
 
 	contentsPrefix = formatTemplate(
 		open('textileTemplatePrefix.txt').read(),
-		level,
-		dir + outname,
+		conf.level,
+		conf.dir + outname,
 		wantComments
 	)
 
 	contentsSuffix = formatTemplate(
 		open('textileTemplateSuffix.txt').read(),
-		level,
-		dir + outname,
+		conf.level,
+		conf.dir + outname,
 		wantComments
 	)
 
 	contents = contentsPrefix + contentsBody + contentsSuffix
 
 	open(outfile, 'w').write(contents)
+	return outfile
 
+codegen_vernum = 0
 
 def dir_textile2html(dir, level = 0):
 	for file in os.listdir('input/' + dir):
+		is_dir = False
+		src_file = 'input/' + dir + file
+		dst_file = None
+		dst_mtime = None
 		if file[-8:] == '.textile':
-			print dir+file
-			textile2html(dir, file, level)
+			conf = CodegenConf( dir, file, level, codegen_vernum )
+			if conf.out_of_date:
+				dst_file = textile2html( conf )
+				dst_mtime = conf.mtime
+				print '   ', dir+file, 'is out of date; generating.'
+			else:
+				print '   ', dir+file, 'is up to date.'
+				continue
 		elif os.path.isdir('input/' + dir + file):
+			is_dir = True
 			if dir != '.' and dir != '..':
 				p = dir + file + '/'
 				try:	os.makedirs('output/'+p)
 				except:	pass
 				dir_textile2html(p, level+1)
 		elif file[-4:] != '.swp' and file != 'Thumbs.db':
-			shutil.copyfile('input/' + dir + file, 'output/' + dir + file)
+			dst_file = 'output/' + dir + file
+			shutil.copyfile('input/' + dir + file, dst_file)
+			dst_mtime = os.stat( src_file ).st_mtime
 
+		if not is_dir:
+			os.utime( dst_file, (dst_mtime,) * 2 )
+
+print 'Generating website data...'
 
 dir_textile2html('')
 
+print 'Copying misc files...'
+
 cmds = [
-	r'cp -R slider output/slider',
-	r'cp *.css *.js output/',
-	r'cp headerBackground.png settingsIcon.png teapot.png output/',
+	r'cp -pR slider output/slider',
+	r'cp -p *.css *.js output/',
+	r'cp -p headerBackground.png settingsIcon.png teapot.png output/',
 ]
 for c in cmds:
+	print '   ', c
 	os.system(c)
 	
+print 'Done.'

File incremental_update.py

+import os
+import sys
+import traceback
+import ssh
+import time
+import pickle
+import threading
+
+# setup logging
+ssh.util.log_to_file('demo_sftp.log')
+os.chdir( 'output' )
+
+def humanize_bytes(bytes, precision=1):
+    abbrevs = (
+        (1<<50L, 'PB'),
+        (1<<40L, 'TB'),
+        (1<<30L, 'GB'),
+        (1<<20L, 'MB'),
+        (1<<10L, 'kB'),
+        (1, 'bytes')
+    )
+    if bytes == 1:
+        return '1 byte'
+    for factor, suffix in abbrevs:
+        if bytes >= factor:
+            break
+    return '%.*f %s' % (precision, bytes / factor, suffix)
+
+def get_local_dirs_files():
+    local_dirs = set([])
+    local_files = {}
+
+    for root_, dirs, files in os.walk(''):
+        root = root_.replace('\\', '/')
+        if root != '':
+            local_dirs.add( root )
+        for fname in files:
+            fpath = ( fname if root == '' else root + '/' + fname )
+            local_files[ fpath ] = os.stat( fpath ).st_mtime
+
+    local_dirs = list(local_dirs)
+    local_dirs.sort()
+
+    return ( local_dirs, local_files )
+
+
+class RemoteStorage(object):
+    def __init__(self):
+        self.t = None
+        self.sftp = None
+        self.file_list_index = '.incremental_updater_file_list'
+
+    def __del__(self):
+        self.t.close()
+
+    def mkdir(self, dpath):
+        try:
+            self.sftp.mkdir( dpath, 0755 )
+        except IOError:
+            pass
+
+    def get_file_list(self):
+        try:
+            return pickle.loads( self.sftp.open( self.file_list_index, 'rb' ).read() )
+        except IOError:
+            return ( {}, {} )
+
+    def save_file_list(self, data):
+        self.sftp.open( self.file_list_index, 'wb' ).write( pickle.dumps(data) )
+
+    def connect(self):
+        # get hostname
+        username = 'h3gdpo'
+        hostname = 'webhost.daily.co.uk'
+        port = 22
+        password = open('../password.txt', 'r').read()
+
+        # get host key, if we know one
+        hostkeytype = None
+        hostkey = None
+        try:
+            host_keys = ssh.util.load_host_keys(os.path.expanduser('~/.ssh/known_hosts'))
+        except IOError:
+            try:
+                # try ~/ssh/ too, because windows can't have a folder named ~/.ssh/
+                host_keys = ssh.util.load_host_keys(os.path.expanduser('~/ssh/known_hosts'))
+            except IOError:
+                print '*** Unable to open host keys file'
+                host_keys = {}
+
+        if host_keys.has_key(hostname):
+            hostkeytype = host_keys[hostname].keys()[0]
+            hostkey = host_keys[hostname][hostkeytype]
+            print 'Using host key of type %s' % hostkeytype
+
+
+        self.t = ssh.Transport((hostname, port))
+        self.t.connect(username=username, password=password, hostkey=hostkey)
+        self.sftp = ssh.SFTPClient.from_transport(self.t)
+        self.sftp.chdir('public_html')
+
+
+
+remote = RemoteStorage()
+remote.connect()
+
+( remote_dirs, remote_files ) = remote.get_file_list()
+( local_dirs, local_files ) = get_local_dirs_files()
+
+for dpath in local_dirs:
+    if dpath not in remote_dirs:
+        remote.mkdir( dpath )
+
+class UploadTask(object):
+    def __init__(self, fpath, reason):
+        self.fpath = fpath
+        self.reason = reason
+        self.size = os.stat( fpath ).st_size
+
+upload_tasks = []
+
+for fpath in local_files.keys():
+    local_mtime = local_files[ fpath ]
+
+    should_copy = False
+    reason = ''
+
+    if fpath not in remote_files:
+        upload_tasks.append( UploadTask( fpath, 'missing ') )
+    elif remote_files[ fpath ] < local_mtime:
+        upload_tasks.append( UploadTask( fpath, 'obsolete') )
+
+total_upload_size = sum( [ f.size for f in upload_tasks ] )
+
+if total_upload_size > 0:
+    def update_progress_bar( item, total_upload_done ):
+        progress_bar_width = 20
+        file_name_width = 40
+        max_line_width = 79
+
+        i = progress_bar_width * total_upload_done / total_upload_size
+        bar = '['
+        bar += '*' * i
+        bar += ' ' * ( progress_bar_width - i )
+        bar += '] '
+        bar += 'uploading %s "' % item.reason
+        if len( item.fpath ) > file_name_width:
+            bar += '...' + item.fpath[-file_name_width+3:]
+        else:
+            bar += item.fpath
+        bar += '"'
+        assert len( bar ) <= max_line_width
+        bar += ' ' * ( max_line_width - len( bar ) )
+        sys.stdout.write( '\r' + bar )
+
+    total_upload_done = 0
+
+    print 'Total upload size:', humanize_bytes( total_upload_size )
+    for t in upload_tasks:
+        update_progress_bar( t, total_upload_done )
+        def progress_func( cur, remaining ):
+            update_progress_bar( t, total_upload_done + cur )
+        remote.sftp.put( t.fpath, t.fpath, progress_func )
+        total_upload_done += t.size
+
+remote.save_file_list( ( local_dirs, local_files ) )
+remote.t.close()
+
+for thread in threading.enumerate():
+    if thread is not threading.currentThread():
+        thread.join()
+
+print '\nUpload complete.'

File input/index.textile

 Hello, world! I'm a programmer obsessed with gamedev, particularly one of these twisted souls who enjoy engine programming more than working on the actual games.
 
-I am currently employed by The Creative Assembly, and work as a graphics programmer on an upcoming console title. Stay tuned, it's going to be shiny! If you'd like to join us, drop me a line; "we're hiring!":http://creative-assembly.com/?p=1316
+I am currently employed at The Creative Assembly, and work as a graphics programmer on an upcoming console title. Stay tuned, it's going to be shiny! If you'd like to join us, drop me a line; "we're hiring!":http://creative-assembly.com/?p=1316
 
 So far, my largest finished hobby project is "Deadlock":code/deadlock/, an arena-based multi-player first person shooter done with four friends from "team0xf".
 
 	max-width: 312px !important;
 	z-index: 1;
 }
+
+p.formula {
+	padding: 0 !important;
+	margin: 0 !important;
+}
+
+p.formula img {
+	border: 1px solid #2c2c2c;
+	padding: 1px !important;
+	background-color: #303030;
+	margin: 0 !important;
+	box-shadow: 6px 4px 10px rgba(0%, 0%, 0%, 0.2) !important;
+	-webkit-box-shadow: 6px 4px 10px rgba(0%, 0%, 0%, 0.2) !important;
+	-moz-box-shadow: 6px 4px 10px rgba(0%, 0%, 0%, 0.2) !important;
+	border-radius: 0 !important;
+	-moz-border-radius: 0 !important;
+}