Commits

Woojong Koh committed b243885

Sync

  • Participants
  • Parent commits 25fced3

Comments (0)

Files changed (31)

 .vim/bundle/a.vim = [git]https://github.com/vim-scripts/a.vim.git
 .vim/bundle/ack.vim = [git]https://github.com/mileszs/ack.vim.git
 .vim/bundle/ctrlp.vim = [git]https://github.com/kien/ctrlp.vim.git
+.vim/bundle/delimitMate = [git]https://github.com/Raimondi/delimitMate.git
 .vim/bundle/grep.vim = [git]https://github.com/vim-scripts/grep.vim.git
 .vim/bundle/gundo.vim = https://bitbucket.org/sjl/gundo.vim
 .vim/bundle/neocomplcache = [git]https://github.com/Shougo/neocomplcache.git
 .vim/bundle/nerdtree = [git]https://github.com/scrooloose/nerdtree.git
 .vim/bundle/python-mode = [git]https://github.com/klen/python-mode.git
 .vim/bundle/snipmate.vim = [git]https://github.com/msanders/snipmate.vim.git
-.vim/bundle/syntastic = [git]https://github.com/scrooloose/syntastic.git
 .vim/bundle/tagbar = [git]git://github.com/majutsushi/tagbar
-.vim/bundle/taglist.vim = [git]https://github.com/vim-scripts/taglist.vim.git
 .vim/bundle/vim-autocomplpop = https://bitbucket.org/ns9tks/vim-autocomplpop
 .vim/bundle/vim-buffergator = [git]https://github.com/jeetsukumaran/vim-buffergator.git
 .vim/bundle/vim-colors-solarized = [git]https://github.com/altercation/vim-colors-solarized.git
 .vim/bundle/vim-commentary = [git]https://github.com/tpope/vim-commentary.git
 .vim/bundle/vim-easymotion = [git]https://github.com/Lokaltog/vim-easymotion.git
-.vim/bundle/vim-l9 = https://bitbucket.org/ns9tks/vim-l9
+.vim/bundle/vim-indent-object = [git]https://github.com/michaeljsmith/vim-indent-object.git
 .vim/bundle/vim-pathogen = [git]https://github.com/tpope/vim-pathogen.git
 .vim/bundle/vim-powerline = [git]https://github.com/Lokaltog/vim-powerline.git
-.vim/bundle/vim-smartinput = [git]https://github.com/kana/vim-smartinput.git
-.vim/bundle/vim-yankstack = [git]https://github.com/maxbrunsfeld/vim-yankstack.git
 .oh-my-zsh = [git]https://github.com/robbyrussell/oh-my-zsh.git
 
 .hgext/hgbb = https://bitbucket.org/birkenfeld/hgbb
 15ae319e069c7773726ca26516e46b8cb4084ec7 .hgext/hgbb
-244a50c40d93873006c875c5d2ae64fcbc987216 .hgext/hgshelve
-09e6c343a6bc597559abb669a261b5d138c9a9dc .hgext/mercurial_keyring
+c3499427318a18f56b648dd92c253f93e422f42f .hgext/hgshelve
+865a10150cab63c5df1f756da15103ce811c6bac .hgext/mercurial_keyring
 30bdbd5f6173041fa3e8726195dc21f8fd6082ea .hgext/onsub
-c2ae9e09ca1f33ff1e13e629a0b2e6bdd19f83a9 .oh-my-zsh
+cf03ee6d884b3850775210acb18cfb669cd666fd .oh-my-zsh
 ca0ce8fcc49a09eca7eb8ff8003554de2f250901 .vim/bundle/DirDiff.vim
 06c1e16a4dcfd1b8b1764e96482af4b063ca7c8a .vim/bundle/MatlabFilesEdition
-698e0c333c1edae8a1d83b04e5674efa7cac3137 .vim/bundle/NrrwRgn
+97709fe7788830312bd2bd20d024654e3a25b3b8 .vim/bundle/NrrwRgn
 91d9da162a4dea283f90ee772470cb2dca927a30 .vim/bundle/OmniCppComplete
 2cbe946206ec622d9d8cf2c99317f204c4d41885 .vim/bundle/a.vim
-9895285042a2fd5691b2f6582aa979e4d1bdffea .vim/bundle/ack.vim
-be5842a376f16c16c5dc4cc1879d7168a074f7de .vim/bundle/ctrlp.vim
+e93f693fd2a9940d6421bf9e4ddd1f535994eaa5 .vim/bundle/ack.vim
+78303a2b5fe4fcfecda0de1c7832fad60859fb63 .vim/bundle/ctrlp.vim
+0326b7c14a8acb7e4185df1f6033d87bab9ff152 .vim/bundle/delimitMate
 1e5658c6d9575cd34be32d0227e06f07c9d58a2e .vim/bundle/grep.vim
 8c570a493d260890242cbdc5a22d8500dc4604cb .vim/bundle/gundo.vim
-73f9a67aca4b5678afc10498a13279db16ce748d .vim/bundle/neocomplcache
-bf79e223aefe0665bcc62b5dcc4c2c23c6f23fc5 .vim/bundle/nerdtree
-e877c7e3e249e5784fcee8feaff7f1a8799335a1 .vim/bundle/python-mode
+f607bd1736bbf1a23bde46a5903925d954f4b289 .vim/bundle/neocomplcache
+c3b63d2fd9c929359231363bcabc880ba29eb96e .vim/bundle/nerdtree
+cd6aec27d127053d18bd2a12713137e41d054208 .vim/bundle/python-mode
 f5a75d075d3c005ebe69e3f5e56cf99516e8aa3b .vim/bundle/snipmate.vim
-ba17657bc409cf58a95d4214f1c703311ba05f72 .vim/bundle/syntastic
 568ef5fd25468a58723b50cf40a48c5dcb46c802 .vim/bundle/tagbar
-53041fbc45398a9af631a20657e109707a455339 .vim/bundle/taglist.vim
-13fe3d8064647e2cdf07e12840108c3f917f5636 .vim/bundle/vim-autocomplpop
+0000000000000000000000000000000000000000 .vim/bundle/vim-autocomplpop
 e3d24db7313c1375d3694e48b62c53ed125f6fbf .vim/bundle/vim-buffergator
 528a59f26d12278698bb946f8fb82a63711eec21 .vim/bundle/vim-colors-solarized
 dc349bb7d30f713d770fc1fa0fe209e6aab82dc8 .vim/bundle/vim-commentary
 667a668e114e9ec0e5d4cbcb0962d835b23614c4 .vim/bundle/vim-easymotion
-3bb534a720fa762aa01d2df2d5d41bd3c4122169 .vim/bundle/vim-l9
+78fffa609b3e6b84ef01ee4c9aba6d7435d7b18e .vim/bundle/vim-indent-object
 96b5726ef3135effd54993e52c773718fb69a95d .vim/bundle/vim-pathogen
 20ab08c9a8fe6cdfb7c303aa3063f211d72f8b2f .vim/bundle/vim-powerline
-78ab4b3df24fa2753d3dfc1be75ed5a3df1565b8 .vim/bundle/vim-smartinput
-02ee85f9624c93a6accf571749dfd7d5e8e2cfab .vim/bundle/vim-yankstack

.ipython/profile_default/startup/00-import.py

+import numpy as np
+import scipy as sp
+
+from pylab import *
+import numpy as np
+import scipy as sp
+
+from pylab import *
+
+
+from IPython.frontend.terminal.ipapp import launch_new_instance
+launch_new_instance()
+raise SystemExit
 # Highlight active window
 set-window-option -g window-status-current-bg red
 #####################################################################
+
+set -g history-limit 10000
 "------------------------------------------------------------
 
 " Customization
+set infercase
+set shiftround
 set nonumber
 set relativenumber
+set gdefault
 set background=dark
 colorscheme solarized
 if has("gui_running")
 endif
 set guioptions-=T	" Remove toolbar
 set guifont=Droid\ Sans\ Mono:h11,Monaco:h12
-
 let mapleader=","
 set scrolloff=2		" Keep some context
+set sidescrolloff=5	" Keep some context
 set incsearch
 "set nowrapscan		" Do not wrap around
 set history=1000
+set viminfo^=!
 set viminfo+=%3		" Save and restore the buffer list
 set clipboard=unnamed
 if has('unnamedplus')
 set macmeta
 set path+=/usr/local/include,/opt/local/include,./include;,./lib;
 set tags+=./tags;,~/.vim/libstdc++_tags
-
+set autoread
+set autowrite
 set backup
 set undofile
-set backupdir=~/.vim/tmp/backup
-set undodir=~/.vim/tmp/undo
 set noswapfile
+set backupdir=~/.vim/tmp/backup//   " include full path
+set undodir=~/.vim/tmp/undo//
+
+" Keep search matches in the middle of the window.
+nnoremap n nzzzv
+nnoremap N Nzzzv
+
+" Easier to type, and I never use the default behavior.
+noremap H ^
+noremap L $
+vnoremap L g_
+
+" Time out on key codes but not mappings.
+" Basically this makes terminal Vim work sanely.
+set notimeout
+set ttimeout
+set ttimeoutlen=10
+set fileformats+=mac
+
+set noignorecase " ignorecase has a problem with tag jump Ctrl-]
+map / /\c
+map ? /\c
+
+" https://github.com/tpope/vim-sensible
+set list
+set showbreak=↪
+if &listchars ==# 'eol:$'
+  set listchars=tab:>\ ,trail:-,extends:>,precedes:<,nbsp:+
+  if &termencoding ==# 'utf-8' || &encoding ==# 'utf-8'
+    let &listchars = "tab:\u21e5 ,trail:\u2423,extends:\u21c9,precedes:\u21c7,nbsp:\u26ad"
+  endif
+endif
 
 runtime macros/matchit.vim	" Enable matchit
 
+
 " Mapping
 nnoremap j gj
 nnoremap k gk
 map <tab> %
 nnoremap <silent> <Leader>. :e .<CR>
+nnoremap D d$   " Made D behave
+
 nnoremap <Leader>a :Ack!<space>
 nnoremap <Leader>gu :GundoToggle<CR>
-nmap <ESC>p <Plug>yankstack_substitute_older_paste
-nmap <ESC>P <Plug>yankstack_substitute_older_paste
-nmap <ESC>n <Plug>yankstack_substitute_newer_paste
-nmap <ESC>N <Plug>yankstack_substitute_newer_paste
 
+" Grep or Ack
+let s:grep_or_ack = 'Grep'
 if executable('ack')
-	nmap <silent> <Leader>* :execute expand('Ack --'.&filetype.' <cword> %')<CR>
-	nmap <silent> <Leader>g* :execute expand('Ack --'.&filetype.' <cword>')<CR>
-elseif executable('grep')
-	nmap <silent> <Leader>* :Grep <cword> %<CR>
-	nmap <silent> <Leader>g* :Grep <cword><CR><CR>
+    let s:grep_or_ack = 'Ack'
 endif
+execute 'nmap <silent> <Leader>* :' . s:grep_or_ack . ' <cword> %<CR>'
+execute 'nmap <silent> <Leader>g* :' . s:grep_or_ack . ' <cword><CR>'
+
 
 " Autocommand
 autocmd BufEnter * silent! lcd %:p:h
 autocmd BufEnter *.tex silent! setlocal textwidth=75 spell spelllang=en_us
 autocmd BufReadPre,BufNewFile SConstruct,Sconscript set filetype=python
 autocmd BufEnter * if filereadable('SConstruct') || filereadable('SConscript') | silent! setlocal makeprg=scons\ -u | else | silent! setlocal makeprg= | endif
-autocmd BufWritePost,FileWritePost * call UpdateTags()
+autocmd VimResized * :wincmd =  " Resize splits when the window is resized
+"autocmd BufWritePost,FileWritePost * call UpdateTags()
 
 function UpdateTags()
+    if !filereadable('tags')
+        return
+    endif
+
     let l:ctags_options = "--sort=foldcase --c++-kinds=+p --fields=+iaS --extra=+q"
     let l:ctags_excludes = '--exclude="*/typeof/*" --exclude="*/preprocessed/*"'
 
     endif
 endfunction
 
-autocmd VimResized * :wincmd =  " Resize splits when the window is resized
-
 " http://vim.wikia.com/wiki/Automatically_open_the_quickfix_window_on_:make
 " Automatically open, but do not go to (if there are errors) the quickfix /
 " location list window, or close it when is has become empty.
 autocmd QuickFixCmdPost [^l]* nested cwindow
 autocmd QuickFixCmdPost    l* nested lwindow
 
+
 " NERDTree
 let NERDTreeChDirMode=2
 let NERDTreeShowBookmarks=1
 let NERDTreeShowHidden=1
 
+
 " CtrlP
-set wildignore+=*.o,*.obj,.DS_Store	" Linux/MacOSX
+set wildignore+=*/tmp/*,*.so,*.swp,*.o,*.obj,.DS_Store    " MacOSX/Linux
+set wildignore+=*\\tmp\\*,*.swp,*.exe   " Windows
 let g:ctrlp_cmd = 'CtrlPMixed'
+let g:ctrlp_mruf_relative = 1
 let g:ctrlp_custom_ignore = '\v[\/]\.(git|hg|svn)$'
 let g:ctrlp_reuse_window = 'netrw\|help\|quickfix\|nerdtree'
 let g:ctrlp_user_command = {
-                \ 'types': {
-                    \ 1: ['.git', 'cd %s && git ls-files'],
-                    \ 2: ['.hg', 'hg --cwd %s locate -I .'],
-                \ }
+            \ 'types': {
+                \ 1: ['.git', 'cd %s && git ls-files'],
+                \ 2: ['.hg', 'hg --cwd %s locate -I .'],
+                \ },
+            \ 'fallback': 'find %s -type f'
             \ }
 
-" Syntastic
-let g:syntastic_mode_map = { 'passive_filetypes': ['cpp'] }
-let g:syntastic_cpp_no_include_search = 1
-let g:syntastic_cpp_compiler_options = ' -std=c++0x'
-let g:syntastic_c_include_dirs = split(&path, ',')
 
 " Tagbar
+autocmd VimEnter * nested :call tagbar#autoopen(1)
+autocmd FileType * nested :call tagbar#autoopen(0)
 autocmd BufEnter * nested :call tagbar#autoopen(0)
 
+
 " NeoComplCache
-let g:acp_enableAtStartup = 0
 let g:neocomplcache_enable_at_startup = 1
+let g:neocomplcache_enable_auto_select = 1
+
 
 " OmniCppComplete
 let OmniCpp_ShowPrototypeInAbbr = 1
 let OmniCpp_MayCompleteScope = 1
 autocmd CursorMovedI,InsertLeave * if pumvisible() == 0 | pclose | endif
+
+
+let g:pymode_breakpoint_key = '<leader>pb'
+let g:pymode_rope = 0
+let g:pymode_rope_guess_project = 0
+let g:pymode_lint_cwindow = 1
+let g:pymode_folding = 0

.weechat/alias.conf

 #
-# alias.conf -- WeeChat v0.3.8
+# alias.conf -- WeeChat v0.3.9
 #
 
 [cmd]

.weechat/aspell.conf

 #
-# aspell.conf -- WeeChat v0.3.8
+# aspell.conf -- WeeChat v0.3.9
 #
 
 [look]
 commands = "ame,amsg,away,command,cycle,kick,kickban,me,msg,notice,part,query,quit,topic"
 default_dict = ""
 during_search = off
+enabled = off
 real_time = off
 word_min_length = 2
 

.weechat/buffers.conf

 #
-# buffers.conf -- WeeChat v0.3.8
+# buffers.conf -- WeeChat v0.3.9
 #
 
 [color]

.weechat/charset.conf

 #
-# charset.conf -- WeeChat v0.3.8
+# charset.conf -- WeeChat v0.3.9
 #
 
 [default]

.weechat/logger.conf

 #
-# logger.conf -- WeeChat v0.3.8
+# logger.conf -- WeeChat v0.3.9
 #
 
 [look]

.weechat/plugins.conf

 #
-# plugins.conf -- WeeChat v0.3.8
+# plugins.conf -- WeeChat v0.3.9
 #
 
 [var]

.weechat/relay.conf

 #
-# relay.conf -- WeeChat v0.3.8
+# relay.conf -- WeeChat v0.3.9
 #
 
 [look]
 raw_messages = 256
 
 [color]
+client = cyan
 status_active = lightblue
 status_auth_failed = lightred
 status_connecting = yellow
 compression_level = 6
 max_clients = 5
 password = ""
+ssl_cert_key = "%h/ssl/relay.pem"
 
 [port]

.weechat/rmodifier.conf

 #
-# rmodifier.conf -- WeeChat v0.3.8
+# rmodifier.conf -- WeeChat v0.3.9
 #
 
 [look]

.weechat/script.conf

+#
+# script.conf -- WeeChat v0.3.9
+#
+
+[look]
+columns = "%s %n %V %v %u | %d | %t"
+diff_color = on
+diff_command = "auto"
+display_source = on
+quiet_actions = on
+sort = "p,n"
+translate_description = on
+use_keys = on
+
+[color]
+status_autoloaded = cyan
+status_held = white
+status_installed = lightcyan
+status_obsolete = lightmagenta
+status_popular = yellow
+status_running = lightgreen
+status_unknown = lightred
+text = default
+text_bg = default
+text_bg_selected = red
+text_date = default
+text_date_selected = white
+text_delimiters = darkgray
+text_description = default
+text_description_selected = white
+text_extension = default
+text_extension_selected = white
+text_name = cyan
+text_name_selected = lightcyan
+text_selected = white
+text_tags = brown
+text_tags_selected = yellow
+text_version = magenta
+text_version_loaded = default
+text_version_loaded_selected = white
+text_version_selected = lightmagenta
+
+[scripts]
+cache_expire = 60
+dir = "%h/script"
+hold = ""
+url = "http://www.weechat.org/files/plugins.xml.gz"

.weechat/weechat.conf

 #
-# weechat.conf -- WeeChat v0.3.8
+# weechat.conf -- WeeChat v0.3.9
 #
 
 [debug]
 command_before_plugins = ""
 display_logo = on
 display_version = on
+sys_rlimit = ""
 
 [look]
 align_end_of_lines = message
 color_inactive_prefix_buffer = on
 color_inactive_time = off
 color_inactive_window = off
+color_nick_offline = off
 color_pairs_auto_reset = 5
 color_real_white = off
 command_chars = ""
 item_time_format = "%H:%M"
 jump_current_to_previous_buffer = on
 jump_previous_buffer_when_closing = on
+jump_smart_back_to_buffer = on
 mouse = on
 mouse_timer_delay = 100
-nickmode = on
-nickmode_empty = off
 paste_bracketed = off
 paste_bracketed_timer_delay = 10
 paste_max_lines = 3
 separator_vertical = ""
 set_title = on
 time_format = "%a, %d %b %Y %T"
+window_separator_horizontal = on
+window_separator_vertical = on
 
 [palette]
 
 chat_inactive_window = darkgray
 chat_nick = lightcyan
 chat_nick_colors = "cyan,magenta,green,brown,lightblue,default,lightcyan,lightmagenta,lightgreen,blue,31,35,38,40,49,63,70,80,92,99,112,126,130,138,142,148,160,162,167,169,174,176,178,184,186,210,212,215,247"
+chat_nick_offline = darkgray
+chat_nick_offline_highlight = default
+chat_nick_offline_highlight_bg = darkgray
 chat_nick_other = cyan
 chat_nick_self = white
 chat_prefix_action = white

.weechat/xfer.conf

 #
-# xfer.conf -- WeeChat v0.3.8
+# xfer.conf -- WeeChat v0.3.9
 #
 
 [look]
 
 # Which plugins would you like to load? (plugins can be found in ~/.oh-my-zsh/plugins/*)
 # Example format: plugins=(rails git textmate ruby lighthouse)
-plugins=(git mercurial autojump brew cloudapp debian macports osx python pip svn vi-mode)
+plugins=(git mercurial autojump brew cloudapp debian macports osx python pip svn vi-mode dircycle cp dirpersisti rsync)
 
 source $ZSH/oh-my-zsh.sh
 
 	. /opt/local/etc/profile.d/autojump.sh
 fi
 
+
+# Python startup file
+export PYTHONSTARTUP=$HOME/.pythonstartup
+
 # virtualenvwrapper
 export WORKON_HOME=$HOME/.virtualenvs
-mkdir -p WORKON_HOME
+mkdir -p $WORKON_HOME
 source virtualenvwrapper.sh
 
 # Use MacVim if it exists
 	alias vim=/Applications/MacVim.app/Contents/MacOS/Vim
 fi
 
+# Back to My Mac (SSH)
+if [ -f ~/Dropbox/Mac\ Sync/.hostnames ]; then
+    source ~/Dropbox/Mac\ Sync/.hostnames
+fi
+
+# MATLAB
+alias matlab="matlab -nodesktop -nosplash"
+alias ssh_i="ssh doyubkim@ssh.intel-research.net -t ssh "
+alias python='python -i'
+
+export REPORTTIME=1
+
 # tmux
 if hash tmux &> /dev/null && [ -z "$TMUX" ]; then
 	SESSION=$USER
 		tmux new-session -s $SESSION
 	fi
 fi
-
-# Back to My Mac (SSH)
-if [ -f ~/Dropbox/Mac\ Sync/.hostnames ]; then
-    source ~/Dropbox/Mac\ Sync/.hostnames
-fi
-
-# MATLAB
-alias matlab="matlab -nodesktop -nosplash"
+# DO NOT ADD ANY CONFIGURATION BELOW HERE.

bin/bootstrap_mac.sh

 sudo port install tig
 sudo port install weechat +aspell +perl +python +tls
 sudo port install wget
+sudo port install sshfs
 sudo /usr/bin/cpan App::Ack
 
 # tmux

bin/encode_imgs_to_vid.sh

+#!/usr/bin/env bash
+
+ffmpeg -r 2 -i frame_%06d.jpg -vcodec libx264 -threads 4 -preset slower -crf 12 -tune animation -pix_fmt yuv420p output_2fps.mp4
+#ffmpeg -r 0.5 -i frame_%06d.jpg -vcodec libx264 -threads 4 -preset slower -crf 12 -tune animation -pix_fmt yuv420p output_60fps.mp4
+#ffmpeg -r 0.25 -i frame_%06d.jpg -vcodec libx264 -threads 4 -preset slower -crf 12 -tune animation -pix_fmt yuv420p output_30fps.mp4
+#ffmpeg -r 90 -i img_%04d.png -vcodec libx264 -threads 4 -preset veryslow -crf 12 -tune animation -pix_fmt yuv420p output_90fps.mp4
+#ffmpeg -r 30 -i img_%04d.png -vcodec libx264 -threads 4 -preset veryslow -crf 12 -tune animation -pix_fmt yuv420p output_30fps.mp4

bin/install_koh.sh

+#!/usr/bin/env bash
+
+sudo pip install -e hg+https://wjkoh@bitbucket.org/wjkoh/koh#egg=koh

bin/merge_pdfs.sh

+#!/usr/bin/env bash
+
+echo gs -dBATCH -dNOPAUSE -q -sDEVICE=pdfwrite -dPDFSETTINGS=/prepress -sOutputFile=out.pdf in1.pdf in2.pdf

bin/pack_python_module.sh

+#!/usr/bin/env bash
+
+SRC_FNAME=$1
+DST_FNAME=${1%.*}
+
+# compress in zip format
+
+# http://sayspy.blogspot.com/2010/03/various-ways-of-distributing-python.html
+echo "#!/usr/bin/env python" | cat - $SRC_FNAME > $DST_FNAME
+chmod u+x $DST_FNAME
+#!/usr/bin/env python 
+#version 2.11.30
+#author tobyrosen@gmail.com
+                                                                                                                                                                                                                                      
+#Copyright (c) 2011,2012, Sony Pictures Imageworks                                                                                                                                                                                                
+#Distributed under the New BSD License.
+#All rights reserved.                                                                                                                                                                                                                        
+#Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:                                                                                              
+#Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.                                                                                                                
+#Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation                                                                                         
+#and/or other materials provided with the distribution.                                                                                                                                                                                      
+#THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,                                                                                           
+#THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS                                                                                       
+#BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF                                                                                                 
+#SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER                                                                                               
+#IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED                                                                                              
+#OF THE POSSIBILITY OF SUCH DAMAGE.                                                                                                                                                                                                          
+        
+
+import optparse
+import sys
+import os
+import time
+import json
+import glob
+import tempfile
+import datetime
+import getpass
+import re
+import math
+
+
+#try to import user customized classes if they exist. default is null class.
+try:
+    from PypCustom import PypCustom
+except ImportError:
+    class PypCustom():    
+        pass
+
+try:
+    from PypCustom import PowerPipeListCustom
+except ImportError :
+    class PowerPipeListCustom():
+        pass
+
+try:
+    from PypCustom import PypStrCustom
+except ImportError  :
+    class PypStrCustom():
+        pass
+        
+try :
+    from PypCustom import PypListCustom
+except ImportError:
+    class PypListCustom():
+        pass
+
+try:
+    from PypCustom import PypFunctionCustom
+except ImportError:
+    class PypFunctionCustom():
+        pass
+
+
+class Colors(object):
+    '''defines basic color scheme'''
+    OFF = chr(27) + '[0m' 
+    RED = chr(27) + '[31m'
+    GREEN = chr(27) + '[32m'
+    YELLOW = chr(27) + '[33m'
+    MAGENTA = chr(27) + '[35m'
+    CYAN = chr(27) + '[36m'
+    WHITE = chr(27) + '[37m'
+    BLUE = chr(27) + '[34m'
+    BOLD = chr(27) + '[1m'
+    COLORS = [OFF, RED, GREEN, YELLOW, MAGENTA, CYAN, WHITE, BLUE, BOLD]
+    
+class NoColors(object):
+    '''defines basic null color scheme'''
+    OFF = ''
+    RED = ''
+    GREEN =''
+    YELLOW = ''
+    MAGENTA = ''
+    CYAN = ''
+    WHITE =''
+    BLUE =  ''
+    BOLD =  ''
+    COLORS = [OFF, RED, GREEN, YELLOW, MAGENTA, CYAN, WHITE, BLUE, BOLD]
+      
+
+
+
+class HistoryObject(object):
+    '''
+    wrapped object that allows history to be stored in container. Used for keeping history
+    when rearrangeing pp lists
+    '''
+    def __init__(self, wrapped_obj):
+        self.__wrapped = wrapped_obj
+        self.current = 0
+
+    def wrapped_object(self):
+        return self.__wrapped
+
+    def set_wrapped_object(self, new_value):
+        self.__wrapped = new_value
+    
+    
+    def __len__(self):
+        try:
+            length = len(self.__wrapped_obj)
+        except : #bouleons etc
+            length = 1
+        
+        return length
+    
+    def __getitem__(self, index):
+        return self.__wrapped[index]
+     
+    def __getattr__(self, name):
+         return getattr(self.__wrapped, name)
+
+    def __cmp__(self, other):
+         return cmp(self.__wrapped, other)
+
+    def __str__(self):
+         return str(self.__wrapped)
+        
+    def __contains__(self, item):
+        if item in self.__wrapped:
+            return True
+        else:
+            return False
+    
+class PowerPipeList(list,PowerPipeListCustom):
+    '''
+    defines pp object, allows manipulation of entire input using python list methods
+    '''
+    def __init__(self, *args):
+        super(PowerPipeList, self).__init__(*args)
+        try:
+            PowerPipeListCustom.__init__(self)
+        except AttributeError:
+            pass
+        self.pyp = Pyp()
+    
+    def stdDev(self,inputs,mean):
+        sumsq=0.0
+        for i in range(len(inputs)):
+            sumsq += (float(inputs[i])-mean) **2
+        return math.sqrt(sumsq/len(inputs))
+    
+    
+    def stats(self):
+        
+        float_pp = [float(x.wrapped_object()) for x in self if x.wrapped_object()]
+      
+        max_pp = max(float_pp)
+        min_pp = min(float_pp)
+        
+        n_pp = len(float_pp)
+        sum_pp = sum(float_pp) 
+        mean_pp = sum_pp / n_pp
+        
+       
+        stddev_pp= self.stdDev(float_pp, mean_pp)
+        
+        stat_pp =[]
+        
+        for p in self:
+            if p.wrapped_object():
+                p.type = dict
+                stat_dict = {}
+                stat_dict['original'] = float(p.wrapped_object())
+                stat_dict['max'] = max_pp
+                stat_dict['min'] = min_pp
+                stat_dict['n'] = n_pp
+                stat_dict['sum'] = sum_pp
+                stat_dict['mean'] = mean_pp
+                stat_dict['stddev'] = stddev_pp
+                
+                p.set_wrapped_object(stat_dict)
+                stat_pp.append(p)
+            
+        return stat_pp
+    
+    
+    
+    
+    
+    def divide(self, n_split):
+        '''
+        splits list into subarrays with n_split members
+        @param n_split: number of members produced by split
+        @type n_split: int
+        @return : new array split up by n_split
+        @rtype : list<str>
+        '''
+        sub_out = []
+        out = []
+        n = 0
+        pyp = Pyp()
+        inputs = self.pyp.flatten_list(self)
+        
+        while inputs:
+            input = inputs.pop(0)
+            n = n + 1
+            sub_out.append(input)
+            if not n % n_split or not inputs:
+                out.append([sub_out])
+                sub_out = []
+        
+        return out
+        
+    def delimit(self, delimiter):
+        '''
+        splits up array based on delimited instead of newlines
+        @param delimiter: delimiter used for split
+        @type delimiter: str
+        @return: new string split by delimiter and joined by ' '
+        @rtype: list<str>
+        '''
+        return ' '.join(self.pyp.flatten_list(self)).split(delimiter)
+
+    def oneline(self,delimiter = ' '):
+        '''
+        combines list to one line with optional delimeter
+        @param delimiter: delimiter used for joining to one line
+        @type delimiter: str
+        @return: one line output joined by delimiter
+        @rtype: list<str>
+        '''
+
+        flat_list =  self.flatten_list(self)
+        return delimiter.join(flat_list)
+
+    def uniq(self):
+        '''
+        returns only unique elements from list
+        @return: unique items
+        @rtype: list<str> 
+        '''
+        strings= self.pyp.flatten_list(self)
+        
+        return list(set(strings))
+
+    def flatten_list(self, iterables):
+        '''
+        returns a list of strings from nested lists
+        @param iterables: nested lists containing strs or PypStrs
+        @type iterables: list
+        @return: unnested list of strings
+        @rtype: list<str>
+        '''
+        return self.pyp.flatten_list(iterables)
+
+    def unlist(self):
+        '''
+        splits a list into one element per line
+        @param self: nested list
+        @type self: list<str>
+        @return: unnested list
+        @rtype: list<str>
+        '''
+        return self.pyp.flatten_list(self)
+
+    
+    
+    def contains(self,target,input):
+        if '__contains__' in dir(input):
+            if target in input:
+                return True
+        
+        else:
+            if target == input:
+                return True
+    
+        return False
+    
+    def after(self, target, after_n=1):
+        '''
+        consolidates after_n lines after matching target text to 1 line
+        @param target: target string to find
+        @type target: str
+        @param after_n: number of lines to consolidate
+        @type after_n: int
+        @return: list of after_n members
+        @rtype: list<str>
+        '''
+        out = []
+        n = 0
+        inputs = self.pyp.flatten_list(self)
+        for input in inputs:
+            n = n + 1
+            if self.contains(target,input):
+                out.append([ [input] + inputs[n:n + after_n] ])
+        
+        return out
+
+    def before(self, target, before_n=1):
+        '''
+        consolidates before_n lines before matching target text to 1 line
+        @param target: target string to find
+        @type target: str
+        @param before_n: number of lines to consolidate
+        @type before_n: int
+        @return: list of before_n members
+        @rtype: list<str>
+        '''
+        out = []
+        n = 0
+        inputs = self.pyp.flatten_list(self)
+        for input in inputs:
+            n = n + 1
+            if self.contains(target,input):
+               out.append([ [input] + inputs[n - before_n - 1:n - 1] ])
+        return out
+        
+
+        
+
+    def matrix(self, target, matrix_n=1):
+        '''
+        consolidates matrix_n lines surrounding matching target text to 1 line
+        @param target: target string to find
+        @type target: str
+        @param matrix_n: number of lines to consolidate
+        @type matrix_n: int
+        @return: list of matrix_n members
+        @rtype: list<str>
+        '''
+        out = []
+        n = 0
+        inputs = self.pyp.flatten_list(self)
+        for input in inputs:
+            n = n + 1
+            if self.contains(target,input):
+                out.append([ inputs[n - matrix_n - 1:n - 1] + [input] + inputs[n:n + matrix_n]  ])
+        return out
+       
+    
+    
+class PypStr(str,PypStrCustom):
+    '''
+    defines p string object, allows manipulation of input line by line using python
+    string methods
+   
+    '''
+    def __init__(self, *args):
+        super(PypStr, self).__init__()       
+        try:
+            PypStrCustom.__init__(self)  
+        except AttributeError:
+            pass
+        
+        
+        try:
+            self.dir = os.path.split(self.rstrip('/'))[0]
+            self.file = os.path.split(self)[1]
+            self.ext = self.split('.')[-1]
+            p.max = 0
+        except:
+            pass
+
+    def trim(self,delim='/'):
+        '''
+        returns everything but the last directory/file
+        @param self: directory path
+        @type self: str
+        @return: directory path missing without last directory/file
+        @rtype: PypStr
+        '''
+        return PypStr(delim.join(self.split(delim)[0:-1]))
+
+
+    def kill(self, *args):
+        '''
+        replaces to_kill with '' in string
+        @param args: strings to remove 
+        @type args : strs
+        @return: string without to_kill
+        @rtype: PypStr
+        '''
+        for arg in args:
+            self = self.replace(arg, '')
+        
+        return PypStr(self)
+    
+    def letters(self):
+        '''
+        returns only letters
+        @return: list of strings with only letters
+        @rtype: PypList
+        '''
+           
+        new_string=''
+        for letter in list(self):
+            if letter.isalpha():
+                new_string = new_string + letter
+            else:
+                new_string = new_string + ' '
+        return [PypStr(x) for x in new_string.split() if x]
+    
+    def punctuation(self):
+        '''
+        returns only punctuation
+        @return: list of strings with only punctuation
+        @rtype: PypList
+        '''
+           
+        new_string=''
+        for letter in list(self):
+            if letter in """!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~""":
+                new_string = new_string + letter
+            else:
+                new_string = new_string + ' '
+        return [PypStr(x) for x in new_string.split() if x]
+    
+    def digits(self):
+        '''
+        returns only digits
+        @return: list of string with only digits
+        @rtype: PypList
+        '''
+        new_string=''
+        for letter in list(self):
+            if letter.isdigit():
+                new_string = new_string + letter
+            else:
+                new_string = new_string + ' '
+        return [PypStr(x) for x in new_string.split() if x]
+        
+    
+    
+    def clean(self,delim = '_'):
+        '''   
+        returns a metacharater sanitized version of input. ignores underscores and slashes and dots.
+        @return: string with delim (default '_') replacing bad metacharacters
+        @rtype: PypStr
+        @param delim: delimeter to rejoin cleaned string with. default is "_"
+        @type delime: str 
+        '''
+        
+        for char in self:
+            if not char.isalnum() and char not in ['/','.',delim]:
+                self = self.replace(char, ' ')
+        return PypStr(delim.join([x for x in self.split() if x.strip()]))
+    
+    def re(self,to_match,group=0):
+        '''   
+        returns characters that match a regex using to_match
+        @return: portion of string that matches regex
+        @rtype: PypStr
+        @param to_match: regex used for matching
+        @type to_match: str 
+        '''
+        
+        match = re.search(to_match,self)
+        if match:
+            return PypStr(match.group(group))
+        else:
+            return ''
+    
+    
+class PypList(list,PypListCustom):
+    '''
+    defines p list object, allows manipulation of input line by line using python
+    list methods
+    '''
+    
+    def __init__(self, *args):
+        super(PypList, self).__init__(*args)
+        try:
+            PypListCustom.__init__(self)
+        except AttributeError:
+            pass
+class Pyp(object):
+    '''
+    pyp engine. manipulates input stream using python methods
+    @ivar history: master record of all manipulations
+    @type history: dict<int:dict>
+    @ivar pwd: current directory
+    @type pwd: str
+    @ivar p: current input line being manipulated
+    @type p: str or list 
+    @ivar n: current input line number
+    @type n: int
+    '''
+    
+    def __init__(self):
+        self.history = {} #dictionary of all data organized input line by input line
+
+        try: #occasionally, python loses pwd info
+            self.pwd = os.getcwd()
+        except:
+            self.pwd =''
+    
+    def get_custom_execute(self):
+        '''returns customized paths to macro files if they are setup'''
+        custom_ob = PypCustom()
+        custom_attrs = dir(custom_ob)
+        
+        if 'custom_execute' in custom_attrs and custom_ob.custom_execute:
+           final_execute = custom_ob.custom_execute
+        else:
+           final_execute = self.default_final_execute
+           
+        return final_execute
+    
+    def default_final_execute(self,cmds,null):
+        '''
+        defines execute command
+        @param cmds: list
+        @param null: str
+        '''
+        for cmd in cmds:
+            os.system(cmd)
+    
+    
+    def get_custom_macro_paths(self):
+        '''returns customized paths to macro files if they are setup'''
+        home =  os.path.expanduser('~')
+        custom_ob = PypCustom()
+        custom_attrs = dir(custom_ob)
+        
+        if 'user_macro_path' in custom_attrs:
+            user_macro_path = custom_ob.user_macro_path
+        else:
+            user_macro_path = home + '/pyp_user_macros.json'
+
+        
+        if 'group_macro_path' in custom_attrs:
+            group_macro_path = custom_ob.group_macro_path
+        else:
+            group_macro_path = home + '/pyp_group_macros.json'
+            
+        return user_macro_path,group_macro_path
+
+    def cmds_split(self, cmds, macros):
+        '''
+        splits total commmand array based on pipes taking into account quotes,
+        parantheses and escapes. returns array of commands that will be processed procedurally.
+        Substitutes macros without executable commands.
+        
+        @param cmds: user supplied command set
+        @type cmds: list<str>
+        @param macros: user defined marcros
+        @type macros: dict<str:dict>
+        @return: list of commands to be evaluated
+        @rtype: list<str>
+        '''
+        cmds = cmds.strip('|')
+        cmd_array = []
+        cmd = ''
+        open_single = False
+        open_double = False
+        open_parenth = 0
+        escape = False
+        letters = list(cmds)
+        while letters:
+            letter = letters.pop(0)
+            if cmd and cmd[-1] == '\\': escape = True
+            
+            #COUNTS QUOTES
+            if letter == "'":
+                if open_single and not escape:
+                    open_single = not open_single
+                else:
+                    open_single = True
+            if letter == '"':
+                if open_double and not escape:
+                    open_double = not open_double
+                else:
+                    open_double = True
+            
+            #COUNTS REAL PARANTHESES
+            if not open_single and not open_double:
+                if letter == '(' :
+                    open_parenth = open_parenth + 1
+                if letter == ')':
+                    open_parenth = open_parenth - 1
+
+            #MONEY MOVE--substitutes command for macro or starts building new command after adding command to cmd_array
+            if cmd.strip() in macros and letter in ['|', '[', '%', ',', '+', ' ']:
+                cmd = cmd.strip()
+                letters = list('|'.join(macros[cmd]['command']) + letter + ''.join(letters))
+                cmd = ''
+            elif letter == '|' and not open_single and not open_double and not open_parenth:#
+                cmd_array.append(cmd)
+                cmd = ''
+            else:
+                cmd = cmd + letter
+            escape = False
+
+            #for last command, either recursively run cmd_split or add last command to array
+        if cmd.strip() in macros and not options.macro_save_name: #allows macro be split and also to be correctly overwritten
+            return self.cmds_split('|'.join(cmd_array + macros[cmd]['command']), macros) #this is by definition the last cmd. 
+        else:
+            cmd_array.append(cmd) #gets last cmd
+
+        
+        return [x for x in cmd_array if x.strip()] #gets rid of extra spaces and nulls
+
+    def load_macros(self,macro_path):
+        '''
+        loads macro file; returns macros dict
+        @param macro_path: file path to macro file
+        @type macro_path: str
+        @return: dictionary of user defined macros
+        @rtype: dict<str:dict>
+        '''
+        #macro_path = self.macro_path
+        if os.path.exists(macro_path):
+            macro_ob = open(macro_path)
+            macros = json.load(macro_ob)
+            macro_ob.close()
+        else:
+           macros = {}
+
+        for macro in macros: #unicode sucks
+            macros[macro][u'command'] = [str(x) for x in macros[macro][u'command'] ]
+        
+        return macros
+
+    def write_macros(self, macros,macro_path, cmds):
+        '''
+        writes macro file
+        @param macros: dictionary of user defined macros
+        @type macros: dict<str:dict>
+        @param macro_path: file path to macro file
+        @type macro_path: str
+        @param cmds: commands to be saved as a macro
+        @type cmds: list<str>
+        '''
+        
+        if options.macro_save_name:
+            macro = options.macro_save_name
+            macro_name = macro.split('#')[0].strip()
+            macros[macro_name] = {}
+            macros[macro_name]['command'] = cmds
+            macros[macro_name]['user'] = getpass.getuser() 
+            macros[macro_name]['date'] =str(datetime.datetime.now()).split('.')[0]
+
+            if '#' in macro: #deals with comments
+                macros[macro_name]['comments'] = '#' + macro.split('#')[1].strip()
+            else:
+                macros[macro_name]['comments'] = ''
+            macro_ob = open(macro_path, 'w')
+            json.dump(macros, macro_ob)
+            macro_ob.close()
+            self.load_macros(macro_path)
+            if macro_name in macros:
+                print Colors.YELLOW + macro_name , "successfully saved!" + Colors.OFF
+                sys.exit()
+            else:
+                print Colors.RED + macro_name, 'was not saved...unknown error!' + Colors.OFF
+                sys.exit(1)
+
+    def delete_macros(self, macros,macro_path):
+        '''
+        deletes macro from file
+        @param macros: dictionary of user defined macros
+        @type macros: dict<str:dict>
+        @param macro_path: file path to macro file
+        @type macro_path: str
+        '''
+        if options.macro_delete_name:
+            if options.macro_delete_name in macros:
+                del macros[options.macro_delete_name]
+                json_ob = open(macro_path, 'w')
+                json.dump(macros, json_ob)
+                json_ob.close()
+                print Colors.MAGENTA + options.macro_delete_name + " macro has been successfully obliterated" + Colors.OFF
+                sys.exit()
+            else:
+                print Colors.RED + options.macro_delete_name + " does not exist" + Colors.OFF
+                sys.exit(1)
+
+    def list_macros(self, macros):
+        '''
+        prints out formated macros, takes dictionary macros as input
+        @param macros: dictionary of user defined macros
+        @type macros: dict<str:dict>
+        '''
+        if options.macro_list or options.macro_find_name:
+            macros_sorted = [x for x in macros]
+            macros_sorted.sort()
+            for macro_name in macros_sorted:
+                if options.macro_list or options.macro_find_name in macro_name or options.macro_find_name in macros[macro_name]['user']:
+                    print Colors.MAGENTA + macro_name + '\n\t ' + Colors.YELLOW+macros[macro_name]['user'] \
+                     + '\t' + macros[macro_name]['date']\
+                    +'\n\t\t' + Colors.OFF + '"'\
+                     + '|'.join(macros[macro_name]['command']) + '"' + Colors.GREEN + '\n\t\t'\
+                      + macros[macro_name].get('comments', '') + Colors.OFF + '\n'
+            sys.exit()
+
+    def load_file(self):
+        '''
+        loads file for pyp processing
+        @return: file data
+        @rtype: list<str>
+        '''
+        if options.text_file:
+            if not os.path.exists(options.text_file):
+                print Colors.RED + options.text_file + " does not exist" + Colors.OFF
+                sys.exit()
+            else:
+                f = [x.rstrip() for x in open(options.text_file) ]
+                return f
+        else:
+            return []
+
+
+    def shell(self, command):
+        '''
+        executes a shell commands, returns output in array sh
+        @param command: shell command to be evaluated
+        @type command: str
+        @return: output of shell command
+        @rtype: list<str>
+        '''
+        sh = [x.strip() for x in os.popen(command).readlines()]
+        return sh
+
+    def shelld(self, command, *args):
+        '''
+        executes a shell commands, returns output in dictionary based on args
+        @param command: shell command to be evaluated
+        @type command: str
+        @param args: optional delimiter. default is ":".
+        @type args: list 
+        @return: hashed output of shell command based on delimiter
+        @rtype: dict<str:str>
+        
+        '''
+        if not args:
+            ofs = ':'
+        else:
+            ofs = args[0]
+        shd = {}
+        for line in [x.strip() for x in os.popen(command).readlines()]:
+            try:
+                key = line.split(ofs)[0]
+                value = ofs.join(line.split(ofs)[1:])
+                shd[key] = value
+            except IndexError:
+                pass
+
+        return shd
+
+    
+    def rekeep(self,to_match):
+        '''
+        keeps lines based on regex string matches
+        @param to_match: regex
+        @type to_match: str
+        @return: True if any of the strings match regex else False
+        @rtype: bool
+        '''
+        
+        match = []
+        flat_p = self.flatten_list(self.p)
+        for item in flat_p:
+            if re.search(to_match,item):
+                match.append(item)
+        if match:
+            return True
+        else:
+            return False
+    
+    def relose(self,to_match):
+        '''
+        loses lines based on regex string matches
+        @param to_match: regex
+        @type to_match: str
+        @return: False if any of the strings match regex else True
+        @rtype: bool
+        '''
+        
+        return not self.rekeep(to_match)
+    
+    
+    def keep(self,*args):
+        '''
+        keeps lines based on string matches
+        @param args: strings to search for 
+        @type args: list<str>
+        @return: True if any of the strings are found else False
+        @rtype: bool
+        '''
+        
+        kept = []
+        for arg in args:
+            flat_p = self.flatten_list(self.p)
+            for item in  flat_p:
+                if arg in item:
+                    kept.append(arg)
+                
+        if kept:
+            return True
+        else:
+            return False
+    
+    def lose(self,*args):
+        '''
+        removes lines based on string matches
+        @param args: strings to search for 
+        @type args: list<str> 
+        @return: True if any of the strings are not found else False
+        @rtype: bool
+        '''
+        return not self.keep(*args)
+
+    def array_tracer(self,input,analysis_mode,power_pipe=''):
+        '''
+        generates colored, numbered output for lists and dictionaries and other types
+        @param input: one line of input from evaluted pyp command
+        @type input: any
+        @param power_pipe: Output from powerpipe (pp) evaluation
+        @type power_pipe: bool
+        @return: colored output based on input contents
+        @rtype: str
+        '''
+        if not input and input not in [0,0.0]: #TRANSLATE FALSES TO EMPTY STRINGS OR ARRAYS. SUPPLIES DUMMY INPUT TO KEEP LINES IF NEEDED.
+            if options.keep_false or power_pipe:
+                input = ' '
+            else:
+                return ''
+       
+       #BASIC VARIABLES
+        nf = 0
+        output = ''
+        
+        if power_pipe:#labels line number of powerpipes
+            n_index = Colors.MAGENTA + '[%s]' % (self.n) + Colors.GREEN
+            final_color = Colors.OFF
+        else: #for line by line
+            n_index = ''
+            final_color=''
+        
+        
+        #DEALS WITH DIFFERENT TYPES OF INPUTS
+        
+        input_type = type(input)
+        
+        if input_type in [ list, PypList, PowerPipeList] :#deals with lists
+            for field in input:
+                if not nf == len(input):
+                    if type(field)  in [str, PypStr]:
+                        COLOR = Colors.GREEN
+                    else:
+                        COLOR = Colors.MAGENTA
+                    output = str(output) + Colors.BOLD + Colors.BLUE + "[%s]" % nf + Colors.OFF + COLOR + str(field) + Colors.GREEN
+                nf = nf + 1
+            final_output =  n_index + Colors.GREEN + Colors.BOLD + '[' + Colors.OFF + output + Colors.GREEN + Colors.BOLD + ']' + Colors.OFF
+
+        elif input_type in [str, PypStr] : #clean output for non-pp operations. this should have no color
+            if analysis_mode: #put colors into output only if it's in analysis mode. 
+                n_index =  Colors.GREEN
+                final_color = Colors.OFF     
+            
+            final_output = n_index +  str(input) + final_color
+        
+        elif input_type in [int, float] :
+            final_output = n_index + Colors.YELLOW + str(input) +  Colors.OFF #had to comppromise here to keep string output clean.
+ 
+        elif input_type is dict: #deals with dictionaries
+            for field in sorted(input,key=lambda x : x):
+                output = output + Colors.OFF + Colors.BOLD + Colors.BLUE  + str(field)  + Colors.GREEN + ": " + Colors.OFF + Colors.GREEN + str(input[field]) + Colors.BOLD + Colors.GREEN + ',\n '
+            final_output = n_index + Colors.GREEN + Colors.BOLD + '{' + output.strip().strip(' ,') + Colors.GREEN + Colors.BOLD + '}' + Colors.OFF
+
+        else: #catches every else
+            final_output =  n_index + Colors.MAGENTA + str(input) + Colors.OFF
+        
+        return final_output
+
+    def cmd_split(self, cmds):
+        '''
+        takes a command (as previously split up by pipes and input as array cmds), 
+        and returns individual terms (cmd_array) that will be evaluated individually.
+        Also returns a string_format string that will be used to stitch together
+        the output with the proper spacing based on the presence of "+" and ","
+        @param cmds: individual commands separated by pipes
+        @type cmds: list<str>
+        @return: individual commands with corresponding string format
+        @rtype: list<str>
+        '''
+       
+        string_format = '%s'
+        cmd_array = []
+        cmd = ''
+        open_quote = False
+        open_parenth = 0
+        open_bracket = 0
+
+        for letter in cmds:
+            
+            if letter in [ "'" , '"']:
+                if cmd and cmd[-1] == '\\':
+                    open_quote = True
+                else:
+                    open_quote = not open_quote
+
+            if not open_quote: #this all ignores text in () or [] from being split by by , or + 
+                if letter == '(' :
+                    open_parenth = open_parenth + 1
+                elif letter == ')':
+                    open_parenth = open_parenth - 1    
+                elif letter == '[' :
+                    open_bracket = open_bracket + 1
+                elif letter == ']':
+                    open_bracket = open_bracket - 1
+                
+                if not open_parenth and not open_bracket and letter in [',', '+']: #these are actual formatting characters
+                    cmd_array.append(cmd)
+                    cmd = ''
+                    string_format = string_format + letter.replace('+', '%s').replace(',', ' %s')
+                    continue
+
+            cmd = cmd + letter
+
+        
+        cmd_array.append(cmd)
+        
+        output = [(cmd_array, string_format)]
+        return output
+
+    def all_meta_split(self, input_str):
+        '''
+        splits a string on any metacharacter
+        @param input_str: input string
+        @type input_str: str
+        @return: list with no metacharacters
+        @rtype: list<str>
+        '''
+
+        for char in input_str:
+            if not char.isalnum():
+                input_str = input_str.replace(char, ' ')
+        return [x for x in input_str.split() if x.strip()]
+
+    def string_splitter(self):
+        '''
+        splits self.p based on common metacharacters. returns a
+        dictionary of this information.
+        @return: input split up by common metacharacters
+        @rtype: dict<str:list<str>>
+        '''
+        
+        whitespace =self.p.split(None)
+        slash =self.p.split('/')
+        underscore =self.p.split('_')
+        colon =self.p.split(':')
+        dot =self.p.split('.')
+        minus =self.p.split('-')
+        all= self.all_meta_split(self.p)
+        comma = self.p.split(',')
+        tab = self.p.split('\t')
+        backslash = self.p.split('\\')
+        
+        
+        split_variables_raw = {
+        
+            'whitespace' :whitespace,
+            'slash' :slash,
+            'underscore' :underscore,
+            'colon' :colon,
+            'dot' :dot,
+            'minus' :minus,
+            'all' : all,
+            'comma' : comma,
+            'tab': tab,
+            'backslash': backslash,
+            
+            'w' :whitespace,
+            's' :slash,
+            'u' :underscore,
+            'c' :colon,
+            'd' :dot,
+            'm' :minus,
+            'a' : all,
+            'mm' : comma,
+            't': tab,
+            'b': backslash,
+                              }
+        #gets rid of empty fields
+        split_variables = dict((x, PypList([PypStr(y) for y in split_variables_raw[x]])) for x in  split_variables_raw)
+        return split_variables
+
+    def join_and_format(self, join_type):
+        '''
+        joins self.p arrays with a specified metacharacter  
+        @param join_type: metacharacter to join array 
+        @type join_type: str
+        @return: string joined by metacharacter
+        @rtype: str
+        '''
+       
+        temp_joins = []
+        derived_string_format = self.history[self.n]['string_format'][-1]
+        len_derived_str_format = len(derived_string_format.strip('%').split('%'))
+
+        if len(self.p) == len_derived_str_format:
+            string_format = derived_string_format #normal output
+            for sub_p in self.p:
+                if type(sub_p) in [list, PypList]:
+                    temp_joins.append(join_type.join(sub_p))
+                else: #deals with printing lists and strings
+                    temp_joins.append(sub_p)
+
+            return PypStr(string_format % tuple(temp_joins))
+
+        else: #deals with piping pure arrays to p
+            return PypStr(join_type.join(PypStr(x)for x in self.p))
+
+    def array_joiner(self):
+        '''
+        generates a dict of self.p arrays joined with various common metacharacters
+        @return: input joined by common metacharacters
+        @rtype: dict<str:str>
+        '''
+        whitespace = self.join_and_format(' ')
+        slash = self.join_and_format(os.sep)
+        underscore = self.join_and_format('_')
+        colon = self.join_and_format(':')
+        dot = self.join_and_format('.')
+        minus = self.join_and_format('-')
+        all = self.join_and_format(' ')
+        comma = self.join_and_format(',')
+        tab = self.join_and_format('\t')
+        backslash = self.join_and_format('\\')
+            
+        join_variables = {
+            'w' : whitespace,
+            's' : slash,
+            'u' : underscore,
+            'c' : colon,
+            'd' : dot,
+            'm' : minus,
+            'a' : all,
+            'mm' : comma,
+            't': tab,
+            'b':backslash,
+            
+            'whitespace' : whitespace,
+            'slash' : slash,
+            'underscore' : underscore,
+            'colon' : colon,
+            'dot' : dot,
+            'minus' : minus,
+            'all' : all,
+            'comma' : comma,
+            'tab': tab,
+            'backslash' : backslash,
+            
+              
+                            }
+
+        return join_variables
+    
+    
+    
+    def generic_variables(self):
+        '''
+        generates a dict of self.p arrays joined with various common metacharacters
+        @return: input joined by common metacharacters
+        @rtype: dict<str:str>
+        '''
+        
+        string_input = str(self.p)
+        whitespace = string_input
+        slash =string_input
+        underscore = string_input
+        colon = string_input
+        dot = string_input
+        minus =string_input
+        all = string_input
+        comma = string_input
+        tab = string_input
+        backslash = string_input
+            
+        generic_variables = {
+            'w' : whitespace,
+            's' : slash,
+            'u' : underscore,
+            'c' : colon,
+            'd' : dot,
+            'm' : minus,
+            'a' : all,
+            'mm' : comma,
+            't': tab,
+            'b':backslash,
+            
+            'whitespace' : whitespace,
+            'slash' : slash,
+            'underscore' : underscore,
+            'colon' : colon,
+            'dot' : dot,
+            'minus' : minus,
+            'all' : all,
+            'comma' : comma,
+            'tab': tab,
+            'backslash' : backslash,
+            
+              
+                            }
+
+        return generic_variables
+
+    def translate_preset_variables(self, translate_preset_variables,file_input, second_stream_input):
+        '''
+        translates variables to protected namespace dictionary for feeding into eval command.
+        @param file_input: data from file
+        @type file_input: list 
+        @param second_stream_input: input from second stream 
+        @type second_stream_input: list<str>
+        @return: values of preset variable for direct use by users
+        @rtype: dict<str:str>
+       '''
+        
+        
+        #generic variables
+        presets = {
+                     'n' : self.kept_n,
+                     'on' : self.n,
+                     'fpp' : file_input,
+                     'spp' : second_stream_input,
+                     'nk': 1000 + self.kept_n,
+                     'shell': self.shell,
+                     'shelld' : self.shelld,
+                     'keep': self.keep,
+                     'lose': self.lose,
+                     'k': self.keep,
+                     'l':self.lose,
+                     'rekeep':self.rekeep,
+                     'relose':self.relose,
+                     'rek':self.rekeep,
+                     'rel':self.relose,
+                     'quote': '"',
+                     'apost':"'",
+                     'qu':'"',
+                     'dollar': '$',
+                     'pwd': self.pwd,
+                     'date': datetime.datetime.now(),
+                     'math':math,
+                     'env': os.environ.get,
+                     'glob' : glob.glob,
+                     'letters': 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ',
+                     'digits': '0123456789',
+                     'punctuation': """!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~""",
+                     'str':(PypStr)
+                             }        
+        
+        #removes nested entries from history
+        history = []
+        for hist in self.history[self.n]['history']:
+            if type(hist) in (list,PypList):
+                hist = self.unlist_p(hist)
+            history.append(hist)
+        presets['history'] = presets['h'] = history
+        
+        # file
+        if options.text_file:
+            try:
+                fp = file_input[self.kept_n]
+            except IndexError:
+                fp = ''
+            presets['fp'] = fp
+
+        # second stream
+        try:
+            sp = second_stream_input[self.kept_n]
+        except IndexError:
+            sp = ''
+        
+        presets['sp'] = sp
+
+        #original input
+        if self.history[self.n]['output']:
+            presets['o'] = self.history[self.n]['history'][0]
+            
+        else:
+            presets['o'] = ''
+        presets['original'] = presets['o']
+         
+        # p cleanup   
+        p = self.p
+        if type(p) in [str]:
+            presets['p'] = PypStr(p)
+        elif  type(p) in [list]:
+            presets['p'] = PypList(p)
+        else:
+            presets['p'] = p
+     
+        #custom functions
+        presets.update(PypFunctionCustom.__dict__) #adds user defined functions
+        return presets
+
+    def blank_history(self):
+        '''
+        initializes history dict for a particular n,
+        where n is the line of the input being processed
+        '''
+        history= {} #creates dict
+        history['error'] = '' # error data
+        history['history'] = [] #  
+        #history['history'].append(self.p) # records current p
+        history['string_format'] = [] #used for formating output
+        history['original_splits'] = {}#dict of original splits
+        history['output'] = True
+        return history
+
+
+    def safe_eval(self, cmd, variables):
+        '''
+        evaluates a str command (cmd) safely. takes a dictionary of protected 
+        namespace variables as input.returns output of python call, which can
+        be any type of python data type (typically str or array though).
+        @param cmd: command to be evaluated using python
+        @type cmd: str
+        @param variables: preset variables used for evaluation
+        @type variables: dictionary
+        @return: output from python evaluation
+        @rtype: list<str>
+        '''
+        if not self.history[self.n]['error'] and self.history[self.n]['output'] is not False: #if no errors, go forward 
+            total_output = []
+            for cm_tuple in self.cmd_split(cmd):#cm_tuple consists of commands and string format.cmd_split splits each command into terms.
+                string_format = cm_tuple[1] #how the results will eventually be formatted.
+                for cm in cm_tuple[0]:#cm is the expression seperated by a + or a ,                        
+                    #evaluate cm and add to dictionary catching any error.
+                    try:
+                        output = eval(cm, variables) #500 lines of code wrap this line!!!
+                    except KeyboardInterrupt:
+                        print Colors.RED + "killed by user" + Colors.OFF
+                        sys.exit()
+                    except IndexError: #ignores index errors like all other shell commands
+                        self.history[self.n]['string_format'].append('%s')
+                        return [''] # this is total_output.                        
+                    except Exception, err:
+                        self.history[self.n]['error'] = Colors.RED + 'error: ' + str(err) + Colors.OFF, Colors.RED + cmd + Colors.OFF
+                        self.history[self.n]['string_format'].append('%s')#adds default string format if error
+                        return ['']
+                        break
+                    #totals output for each cm
+                    try:
+                        if output is True : #allows truth tests
+                            output = self.p
+                    except:
+                        pass  
+                    total_output.append(output)
+                self.history[self.n]['string_format'].append(string_format)
+
+            
+            return total_output
+
+    
+    def get_user_input(self, total_output,second_stream_input,file_input, power_pipe):
+        '''
+        figures out what to show user in terms of powerpipe output. does NOT update history dictionary.
+        
+        @param total_output: line output from eval
+        @type total_output: list<string>
+        @param second_stream_input: entire input from second string
+        @type second_stream_input: list<string>
+        @param file_input: entire input from file
+        @type file_input: list<string>
+        @param power_pipe: kind of power pipe
+        @type power_pipe: string
+        @return: output for display
+        @rtype: list<str>
+        '''
+    
+        try: #who knows what could happen with user input
+            n = self.n
+            if  power_pipe ==    'pp' and total_output or not power_pipe: #standard input
+                user_output =  total_output
+            elif power_pipe ==   'spp'  and second_stream_input:
+                user_output =  [second_stream_input[n]]
+            elif power_pipe ==   'fpp' and file_input:
+                user_output =  [file_input[n]]
+            elif power_pipe: #  power pipe variable is referenced, but does not exist.
+                print Colors.RED + "YOU'RE LIST VARIABLE DOES NOT EXIST: " + Colors.GREEN + power_pipe + Colors.OFF
+                sys.exit()
+        except: #default output is null per line
+            user_output =['']   
+        return user_output
+    
+    def update_history(self, total_output,second_stream_input,file_input, power_pipe):
+        '''
+        updates history dictionary with output from python evaluation
+        @param total_output: line output from eval
+        @type total_output: list<string>
+        @param second_stream_input: entire input from second string
+        @type second_stream_input: list<string>
+        @param file_input: entire input from file
+        @type file_input: list<string>
+        @param power_pipe: kind of power pipe
+        @type power_pipe: string
+        '''
+        #marks null output as False EXCEPT when output is zero, powerpipe. Null output is not printed out.
+        if (total_output in [False]\
+            or  [x for x in self.flatten_list(total_output) if x is False ] == total_output )\
+            and not power_pipe:
+            
+        #make irrelevent data false
+            self.history[self.n]['history'].append(False)
+            self.history[self.n]['output']=False
+        else: # good output
+            string_format = self.history[self.n]['string_format'][-1]
+            output_array = []
+            history_array = []
+            contains_list = False
+            #actual output is p or pp unless spp is invoked.
+            user_input = self.get_user_input(total_output, second_stream_input, file_input, power_pipe)
+        
+            #update history array
+            
+            contains_list = False
+            for out in total_output: # forms an array called_output array of strings or array_traced strings
+                history_array.append(out) # for feeding back to pipe
+                if type(out)  in [list, PypList]:
+                    contains_list = True 
+            
+            contains_number = True if [x for x in total_output if type(x) in [int,float]] == total_output else False
+            contains_dict = True if [x for x in total_output if type(x) in [dict]] == total_output else False
+            #toggle analysis mode if not all input is strings
+        
+            if [x for x in total_output if type(x) in [str,PypStr]] == total_output:
+                analysis_mode = False
+            else:
+                analysis_mode = True
+            
+            #update actual output
+            for out in user_input:
+                traced_output = self.array_tracer(out,analysis_mode, power_pipe)
+                output_array.append(traced_output) # for output
+
+            self.history[self.n]['output'] = string_format % (tuple(output_array))
+            
+            if contains_list or contains_number or contains_dict:# or format_mismatch: #this section prevents buildup of recursive lists.
+                self.history[self.n]['history'].append(total_output) # just adds list to total output if list
+            else:
+                self.history[self.n]['history'].append(string_format % (tuple(history_array))) # adds properly formatted string if string.
+    
+    def flatten_list(self, iterables):
+        '''
+        returns a list of strings from nested lists
+        @param iterables: nested list to flatten
+        @type iterables: list<str>
+        '''
+        out = []
+        if '__iter__' in dir(iterables) or type(iterables) in [list, PowerPipeList,tuple,PypList]:
+            #expand history objects
+            if [x for x in iterables if type(x) in [HistoryObject]]:            
+            
+                expanded_iterables = []
+                for iterable in iterables:
+                    if type(iterable) in [HistoryObject]:
+                        iterable = iterable.wrapped_object()
+                    expanded_iterables.append(iterable)
+                    iterables = expanded_iterables
+            #if [x for x in iterables if type(x) in [str, PypStr,HistoryObject,type,int,float]]:
+
+            if [x for x in iterables if type(x) not in [list,PowerPipeList,tuple,PypList]]: #str,int,etc
+                out = out + iterables #add the lists and be done
+
+            else:
+                for x in iterables:
+                    out = out + self.flatten_list(x)
+        else:              #catches non iterables
+            out = [iterables] #non-iterables
+        return out
+    
+    
+    
+    def stamp_history(self,power_pipe_inputs, new_inputs):
+        '''
+        makes new object stamped with all history info
+        @param power_pipe_inputs: PowerPipeList
+        @param new_inputs: Booleon
+        '''
+        history_inputs= []
+        input_index = 0
+
+        for input in power_pipe_inputs:
+            
+            input_type = type(input)
+            input_copy = input
+            input = HistoryObject(input)
+            if input_index in self.history and  self.history[input_index]['output'] is False: #ignore false
+                input_index = input_index + 1 
+                continue
+            elif not new_inputs and input_index in self.history: #stamps inputs already in history
+                    input.history = self.history[input_index] #get from history dict
+            else:
+                input.history = self.blank_history()      #new input. make fresh entry
+                input.history['history'] = [input_copy]
+                input.history['string_format'] = ['%s'] 
+            input.type = input_type
+            history_inputs.append(input)
+            input_index = input_index + 1 
+        return PowerPipeList(history_inputs)
+    
+    
+    def power_pipe_sum(self, ints): 
+        '''
+        overloaded sum operator. ignores non-digit strings. converts everything else to ints or floats
+        @param ints: varies
+        '''
+        
+        new_ints = []
+        for line in ints:
+            if 'history' in dir(line):
+                line = line.wrapped_object()
+            if type(line) in [int,float]:
+                new_ints.append(line)
+            elif type(line) in [str, PypStr]:
+                if line.isdigit(): 
+                    new_ints.append(int(line))
+                elif line.replace('.','').isdigit():
+                    new_ints.append(float(line))
+        return sum(new_ints)
+    
+    
+    
+    def power_pipe_eval(self, cmd, inputs, second_stream_input, file_input, power_pipe_type):
+        '''
+        evaluates pp statement. returns sanitized result.
+        @param cmd: power pipe command
+        @type cmd: str
+        @param inputs: inputs from std-in or previous python eval
+        @type inputs: list<str>
+        @param power_pipe_type: kind of powerpipe 
+        @type power_pipe_type: str
+        @return: 'p' and output of python evaluation
+        @rtype: list<str>
+        '''
+        variables = {}
+        padded_output = []
+       
+        variables['str'] = PypStr #useful for list comps
+        
+        inputs = self.flatten_list(inputs)
+
+        
+        variables['pp'] = self.stamp_history(inputs, False)
+        variables['spp'] = self.stamp_history(second_stream_input, True)
+        variables['fpp'] = self.stamp_history(file_input, True)
+        variables['sum'] = self.power_pipe_sum
+        variables['math'] = math 
+       
+        
+         
+        try:
+            output = eval(cmd, variables) #1000 lines of code wrap this line!!!
+        except KeyboardInterrupt:
+            print Colors.RED + "killed by user" + Colors.OFF
+            sys.exit()
+        except Exception, err:
+            print Colors.RED + 'error: ' + str(err) + Colors.OFF, Colors.RED + cmd + Colors.OFF
+            sys.exit()
+        
+        if output is None: #allows use of inplace methods like sort
+            output = variables[power_pipe_type]
+        
+        if type(output)  in [int, float]: #keeps output in array
+            output = [output]
+
+        if type(output) in [HistoryObject]: #makes sure output is in list of lists
+            output = [output]
+        
+        if type(output) in [str, PypStr,tuple,type]: #makes sure output is in list of lists
+            output = [[output]]
+        
+        if [x for x in output if type(x) in [tuple]]:#changes tuples to lists 
+            output = [PypList(x) for x in output if type(x) in [tuple]]
+     
+      
+            
+        history_output = self.restore_history(output)         
+    
+        if len(history_output) == 1:      #turn off powerpipe if output is single item
+            power_pipe_type = ''      
+
+        return history_output,  power_pipe_type
+     
+        
+    def restore_history(self, output):  
+        '''
+        restores history dictionary to reflect changes. recasts all objects to their original form.
+        '''
+        
+        self.history = {}
+        # try to preserve history...if this fails, we go with zero history
+        output_index = 0
+        history_output = []
+        for out in output:  
+            if 'history' in dir(out):
+                history =  out.history
+                out = self.recursive_recast(out) #recast back 
+            else: #non stamped and deeply stamped both need new entries
+                out = self.recursive_recast(out) #recast right away
+                history = self.blank_history()     #new input. make fresh entry
+                history['history'] = [self.unlist_p(out)]
+                history['string_format'] = ['%s'] 
+            
+            history_output.append(out)
+            self.history[output_index] = history
+            output_index = output_index + 1
+        
+        return history_output
+  
+    
+    def recursive_recast(self, out):
+        '''
+        takes nested list. recasts all objects to their original type if they are stamped
+        @param out: varies
+        '''
+        try:
+            if 'type' in dir(out): #its stamped
+                return out.type(out)
+            elif type(out) in [list,PowerPipeList]: #try to return list
+                return [self.recursive_recast(x) for x in out]
+
+            elif type(out) in [tuple]: 
+                return ' '.join(out)                #user is using commas to join items with spaces
+            else:                                   #for everything else.
+                return out
+        except:                          
+            return out                              #who knows what people will try.
+    
+    
+    
+    
+    def detect_power_pipe(self, command, power_pipe_type):
+        '''
+        detects presense of powerpipe
+        @param command: command to be evaluated
+        @type command: str
+        @param power_pipe_type: kind of powerpipe (future use)
+        @type power_pipe_type: str
+        @return: True if powerpipe else False
+        @rtype: bool
+        '''
+        open_quote = False
+        cmd_raw = list(command)
+        cmd = []
+
+        for letter in cmd_raw:
+            if letter not in  ['"', "'"] and not letter.isalnum():
+                letter = ' '
+            cmd.append(letter)
+
+         
+        cmds = ''.join(cmd).split()
+        for cmd in cmds:
+            cmd = list(cmd)
+            test_cmd = ''
+            while cmd:
+                letter = cmd.pop(0)
+                test_cmd = test_cmd + letter
+                if not open_quote:
+                    if  power_pipe_type == test_cmd and not cmd:
+                        return True
+                    
+                if letter in [ "'" , '"']:
+                    if cmd and cmd[0] == '\\':
+                        open_quote = True
+                    else:
+                        open_quote = not open_quote
+        return False
+
+    def format_input(self, cmd, input_set, second_stream_input, file_input):
+        '''
+        checks for powerpipe presence, evaluates powerpipe pp and returns 
+        formatted output if detected
+        @param cmd: user command
+        @type cmd: str
+        @param input_set: input from std-in or previous python evaluation
+        @type input_set: list<str>
+        @return: command, input set, presence of powerpipe
+        @rtype: list<str>
+        '''