Commits

yang xiaoyong committed 3b2d13b Draft

add py

Comments (0)

Files changed (10)

html/js/example-1.html

+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
+<html lang="en">
+<head>
+  <meta http-equiv="Content-Type" content="text/html;charset=UTF-8">
+  <title></title>
+</head>
+<body>
+  <script type="text/javascript">
+    document.write("<h1>The date is " + (new Date()).toDateString() + "</h1>");
+  </script>
+</body>
+</html>
+import os
+import subprocess
+import itertools
+
+MAIL_CONFIG = {}
+SEND_MAIL = {'sendmail_path': '/usr/sbin/sendmail'}
+
+def dictadd(*dicts):
+    """
+    Returns a dictionary consisting of the keys in the argument dictionaries.
+    If they share a key, the value from the last argument is used.
+
+        >>> dictadd({1: 0, 2: 0}, {2: 1, 3: 1})
+        {1: 0, 2: 1, 3: 1}
+    """
+    result = {}
+    for dct in dicts:
+        result.update(dct)
+    return result
+
+def safestr(obj, encoding='utf-8'):
+    r"""
+    Converts any given object to utf-8 encoded string.
+
+        >>> safestr('hello')
+        'hello'
+        >>> safestr(u'\u1234')
+        '\xe1\x88\xb4'
+        >>> safestr(2)
+        '2'
+    """
+    if isinstance(obj, unicode):
+        return obj.encode(encoding)
+    elif isinstance(obj, str):
+        return obj
+    elif hasattr(obj, 'next'): # iterator
+        return itertools.imap(safestr, obj)
+    else:
+        return str(obj)
+
+
+class _EmailMessage:
+    def __init__(self, from_address, to_address, subject, message, headers=None, **kw):
+        def listify(x):
+            if not isinstance(x, list):
+                return [safestr(x)]
+            else:
+                return [safestr(a) for a in x]
+
+        subject = safestr(subject)
+        message = safestr(message)
+
+        from_address = safestr(from_address)
+        to_address = listify(to_address)
+        cc = listify(kw.get('cc', []))
+        bcc = listify(kw.get('bcc', []))
+        recipients = to_address + cc + bcc
+
+        import email.Utils
+        self.from_address = email.Utils.parseaddr(from_address)[1]
+        self.recipients = [email.Utils.parseaddr(r)[1] for r in recipients]
+
+        self.headers = dictadd({
+          'From': from_address,
+          'To': ", ".join(to_address),
+          'Subject': subject
+        }, headers or {})
+
+        if cc:
+            self.headers['Cc'] = ", ".join(cc)
+
+        self.message = self.new_message()
+        self.message.add_header("Content-Transfer-Encoding", "7bit")
+        self.message.add_header("Content-Disposition", "inline")
+        self.message.add_header("MIME-Version", "1.0")
+        self.message.set_payload(message, 'utf-8')
+        self.multipart = False
+
+    def new_message(self):
+        from email.Message import Message
+        return Message()
+
+    def attach(self, filename, content, content_type=None):
+        if not self.multipart:
+            msg = self.new_message()
+            msg.add_header("Content-Type", "multipart/mixed")
+            msg.attach(self.message)
+            self.message = msg
+            self.multipart = True
+
+        import mimetypes
+        try:
+            from email import encoders
+        except:
+            from email import Encoders as encoders
+
+        content_type = content_type or mimetypes.guess_type(filename)[0] or "applcation/octet-stream"
+
+        msg = self.new_message()
+        msg.set_payload(content)
+        msg.add_header('Content-Type', content_type)
+        msg.add_header('Content-Disposition', 'attachment', filename=filename)
+
+        if not content_type.startswith("text/"):
+            encoders.encode_base64(msg)
+
+        self.message.attach(msg)
+
+    def prepare_message(self):
+        for k, v in self.headers.iteritems():
+            if k.lower() == "content-type":
+                self.message.set_type(v)
+            else:
+                self.message.add_header(k, v)
+
+        self.headers = {}
+
+    def send(self):
+        self.prepare_message()
+        message_text = self.message.as_string()
+
+        if MAIL_CONFIG:
+            server = MAIL_CONFIG.get('smtp_server')
+            port = MAIL_CONFIG.get('smtp_port', 0)
+            username = MAIL_CONFIG.get('smtp_username')
+            password = MAIL_CONFIG.get('smtp_password')
+            debug_level = MAIL_CONFIG.get('smtp_debuglevel', None)
+            starttls = MAIL_CONFIG.get('smtp_starttls', False)
+
+            import smtplib
+            smtpserver = smtplib.SMTP(server, port)
+
+            if debug_level:
+                smtpserver.set_debuglevel(debug_level)
+
+            if starttls:
+                smtpserver.ehlo()
+                smtpserver.starttls()
+                smtpserver.ehlo()
+
+            if username and password:
+                smtpserver.login(username, password)
+
+            smtpserver.sendmail(self.from_address, self.recipients, message_text)
+            smtpserver.quit()
+        #elif webapi.config.get('email_engine') == 'aws':
+            #import boto.ses
+            #c = boto.ses.SESConnection(
+              #aws_access_key_id=webapi.config.get('aws_access_key_id'),
+              #aws_secret_access_key=web.api.config.get('aws_secret_access_key'))
+            #c.send_raw_email(self.from_address, message_text, self.from_recipients)
+        else:
+            sendmail = SEND_MAIL.get('sendmail_path', '/usr/sbin/sendmail')
+
+            assert not self.from_address.startswith('-'), 'security'
+            for r in self.recipients:
+                assert not r.startswith('-'), 'security'
+
+            cmd = [sendmail, '-f', self.from_address] + self.recipients
+
+            if subprocess:
+                p = subprocess.Popen(cmd, stdin=subprocess.PIPE)
+                p.stdin.write(message_text)
+                p.stdin.close()
+                p.wait()
+
+    def __repr__(self):
+        return "<EmailMessage>"
+
+    def __str__(self):
+        return self.message.as_string()
+
+def sendmail(from_address, to_address, subject, message, headers=None, **kw):
+    """
+    Sends the email message `message` with mail and envelope headers
+    for from `from_address_` to `to_address` with `subject`.
+    Additional email headers can be specified with the dictionary
+    `headers.
+
+    Optionally cc, bcc and attachments can be specified as keyword arguments.
+    Attachments must be an iterable and each attachment can be either a
+    filename or a file object or a dictionary with filename, content and
+    optionally content_type keys.
+
+    If `web.config.smtp_server` is set, it will send the message
+    to that SMTP server. Otherwise it will look for
+    `/usr/sbin/sendmail`, the typical location for the sendmail-style
+    binary. To use sendmail from a different path, set `web.config.sendmail_path`.
+    """
+    attachments = kw.pop("attachments", [])
+    mail = _EmailMessage(from_address, to_address, subject, message, headers, **kw)
+
+    for a in attachments:
+        if isinstance(a, dict):
+            mail.attach(a['filename'], a['content'], a.get('content_type'))
+        elif hasattr(a, 'read'): # file
+            filename = os.path.basename(getattr(a, "name", ""))
+            content_type = getattr(a, 'content_type', None)
+            mail.attach(filename, a.read(), content_type)
+        elif isinstance(a, basestring):
+            f = open(a, 'rb')
+            content = f.read()
+            f.close()
+            filename = os.path.basename(a)
+            mail.attach(filename, content, None)
+        else:
+            raise ValueError, "Invalid attachment: %s" % repr(a)
+
+    mail.send()

py/pinyin-ascii.py

+# -*- encoding:utf-8 -*-
+
+ascii_gbk_maps = {
+    'a': (-20319, -20284),
+    'b': (-20283, -19776),
+    'c': (-19775, -19219),
+    'd': (-19218, -18711),
+    'e': (-18710, -18527),
+    'f': (-18526, -18240),
+    'g': (-18239, -17923),
+    'h': (-17922, -17418),
+    'j': (-17417, -16475),
+    'k': (-16474, -16213),
+    'l': (-16212, -15641),
+    'm': (-15640, -15166),
+    'n': (-15165, -14923),
+    'o': (-14922, -14915),
+    'p': (-14914, -14631),
+    'q': (-14630, -14150),
+    'r': (-14149, -14091),
+    's': (-14090, -13119),
+    't': (-13118, -12839),
+    'w': (-12838, -12557),
+    'x': (-12556, -11848),
+    'y': (-11847, -11056),
+    'z': (-11055, -10247),
+}
+
+def belongs_to(value, range_start, range_end):
+    if range_start <= value <= range_end:
+        return True
+    return False
+
+def pinyin_to_ascii(unicode_str):
+    gbk_str = unicode_str.encode('gbk')
+    if len(gbk_str) == 1:
+        return gbk_str
+    value = ord(gbk_str[0]) * 256 + ord(gbk_str[1]) - 65536
+    for k, ranges in ascii_gbk_maps.iteritems():
+        if belongs_to(value, *ranges):
+            return k
+    return ''
+
+def test():
+    s = u'谁'
+    s = u'你'
+    s = u'g'
+    s = u'帝'
+    print pinyin_to_ascii(s)
+
+test()
+def multi_get_letter(str_input):
+
+    if isinstance(str_input, unicode):
+        unicode_str = str_input
+    else:
+        try:
+            unicode_str = str_input.decode('utf8')
+        except:
+            try:
+                unicode_str = str_input.decode('gbk')
+            except:
+                print 'unknown coding'
+                return
+
+    return_list = []
+    for one_unicode in unicode_str:
+        print single_get_first(one_unicode)
+        return_list.append(single_get_first(one_unicode))
+    return "".join(return_list)
+
+def single_get_first(unicode1):
+    str1 = unicode1.encode('gbk')
+    try:
+        ord(str1)
+        return str1
+    except:
+        asc = ord(str1[0]) * 256 + ord(str1[1]) - 65536
+        if asc >= -20319 and asc <= -20284:
+            return 'a'
+        if asc >= -20283 and asc <= -19776:
+            return 'b'
+        if asc >= -19775 and asc <= -19219:
+            return 'c'
+        if asc >= -19218 and asc <= -18711:
+            return 'd'
+        if asc >= -18710 and asc <= -18527:
+            return 'e'
+        if asc >= -18526 and asc <= -18240:
+            return 'f'
+        if asc >= -18239 and asc <= -17923:
+            return 'g'
+        if asc >= -17922 and asc <= -17418:
+            return 'h'
+        if asc >= -17417 and asc <= -16475:
+            return 'j'
+        if asc >= -16474 and asc <= -16213:
+            return 'k'
+        if asc >= -16212 and asc <= -15641:
+            return 'l'
+        if asc >= -15640 and asc <= -15166:
+            return 'm'
+        if asc >= -15165 and asc <= -14923:
+            return 'n'
+        if asc >= -14922 and asc <= -14915:
+            return 'o'
+        if asc >= -14914 and asc <= -14631:
+            return 'p'
+        if asc >= -14630 and asc <= -14150:
+            return 'q'
+        if asc >= -14149 and asc <= -14091:
+            return 'r'
+        if asc >= -14090 and asc <= -13119:
+            return 's'
+        if asc >= -13118 and asc <= -12839:
+            return 't'
+        if asc >= -12838 and asc <= -12557:
+            return 'w'
+        if asc >= -12556 and asc <= -11848:
+            return 'x'
+        if asc >= -11847 and asc <= -11056:
+            return 'y'
+        if asc >= -11055 and asc <= -10247:
+            return 'z'
+        return ''
+#!/usr/bin/env python
+# -*- encoding:utf8 -*-
+
+import os
+import sys
+import datetime
+import csv
+import re
+import operator
+from collections import defaultdict
+
+from utils import zipfiles, sendmail
+from config import CWD, MAIL_CONFIG, MAIL_LIST, day_fmt, to_gbk
+
+UPLOAD_DIR = '/home/backups/uploads/game/'
+HOST_IDS = os.listdir(UPLOAD_DIR)
+
+today = datetime.datetime.today()
+yesterday = today - datetime.timedelta(1)
+
+class SumData(object):
+    def __init__(self, csv_files, rules={}, has_header=True):
+        self.csv_files = csv_files
+        self.has_header = has_header
+        self.data = defaultdict(int)
+        self.rules = rules
+        self.titles = []
+        self.get_titles()
+        self.sum_data()
+
+    def get_titles(self):
+        filename = self.csv_files[0]
+        with open(filename, 'rb') as fb:
+            csv_reader = csv.reader(fb)
+            if self.has_header:
+                header = csv_reader.next()
+                self.titles.append(header)
+            for row in csv_reader:
+                self.titles.append(row[0])
+
+    def sum_data(self):
+        for csv_file in self.csv_files:
+            with open(csv_file, 'rb') as fb:
+                csv_reader = csv.reader(fb)
+                print 'read %s' % csv_file
+                if self.has_header:
+                    row = csv_reader.next()
+                for row in csv_reader:
+                    self.data[row[0]] += int(row[1])
+
+    def as_csv(self, csv_file):
+        '''output as csv file'''
+        rules = self.rules
+        with open(csv_file, 'wb') as fb:
+            writer = csv.writer(fb)
+            titles = self.titles
+
+            if self.has_header:
+                writer.writerow(titles[0])
+                titles = titles[1:]
+
+            for title in titles:
+                row = [title, self.data[title]]
+
+                if not title in rules:
+                    value = 0
+                    for _title, rule in rules.iteritems():
+                        if callable(rule):
+                            if rule(self.data[title]):
+                                den = self.data[_title]
+                                if den == 0:
+                                    print >> sys.stderr, "ZeroDivisionError: %s" % title.decode('gb18030')
+                                else:
+                                    value = '%.2f%%' % (self.data[title] * 1.0 / den * 100)
+                        elif isinstance(rule, dict):
+                            value = rule.get('value', 0)
+                        else:
+                            raise TypeError('rule only support dict, func type!')
+
+                    row.append(value)
+                writer.writerow(row)
+
+class SumMoreCsv(object):
+    def __init__(self, csv_files, fields=None, has_header=True):
+        self.fields = fields
+        self.has_header = has_header
+        self._data = [self.csv_to_dict(f) for f in csv_files]
+        self.titles = []
+        self.data = {}
+
+    def csv_to_dict(self, csv_file):
+        data = {}
+        with open(csv_file, 'rb') as fb:
+            csv_reader = csv.reader(fb)
+
+            if self.has_header:
+                csv_reader.next()
+
+            for row in csv_reader:
+
+                if self.fields is not None:
+                    if not row[0] in self.fields:
+                        continue
+
+                if row[0] not in self.titles:
+                    self.titles.append(row[0])
+
+                data[row[0]] = int(row[1])
+
+        return data
+
+    def sum(self, oper='add'):
+        for title in self.titles:
+            self.data = reduce(getattr(operator, oper),
+                               [d.get(title, 0) for d in self._data])
+        return self.data
+
+def main():
+    ids = []
+    csv_files = []
+    output_file = os.path.join(CWD, 'report.zip')
+    for host_id in HOST_IDS:
+        csv_dir = os.path.join(UPLOAD_DIR, host_id, today.strftime('%Y%m%d'), 'res')
+        if not os.path.exists(csv_dir):
+            print "[WARN] can not found directory: %s" % (csv_dir)
+            continue
+        files = [os.path.join(csv_dir, f) for f in os.listdir(csv_dir)
+                 if f.endswith('.csv') and re.search(r'\d+-%s' % yesterday.strftime(day_fmt), f)]
+        if files:
+            ids.append(host_id)
+
+        csv_files.extend(files)
+
+    ids.sort()
+    is_economy = lambda x: re.compile(r'^economy').search(x)
+    sum_files = [os.path.join(CWD, 'game_sum.csv'),
+                 os.path.join(CWD, 'economy_sum.csv'),]
+
+    # 游戏玩法统计
+    game_sum = SumData([f for f in csv_files if not is_economy(os.path.basename(f))],
+                       rules={to_gbk(u'登录uid数'): lambda x: True})
+
+    game_sum.as_csv(sum_files[0])
+
+    # 经济系统统计
+    economy_sum = SumData([f for f in csv_files if is_economy(os.path.basename(f))],
+                          rules={to_gbk(u'总回收'): lambda x: x <= 0,
+                                 to_gbk(u'总产出'): lambda x: x >0,
+                                 to_gbk(u'金钱当天结余'): lambda x: False})
+
+    economy_sum.as_csv(sum_files[1])
+
+    csv_files.extend(sum_files)
+    print csv_files
+    zipfiles(output_file, csv_files)
+
+    has_send = False
+    retry_times = 3
+    while retry_times > 0:
+        retry_times -= 1
+
+        try:
+            sendmail(MAIL_CONFIG['smtp_username'],
+                     MAIL_LIST,
+                     '%s 日志扫档统计报告' % yesterday.strftime(day_fmt),
+                     'Hi, all\n\n 每天日志扫档汇总,详细请下载邮件附件查看。\n\n 扫描服务器列表: \n%s ' % '\n'.join(ids),
+                     attachments=[output_file])
+            has_send = True
+        except Exception, e:
+            print >> sys.stderr, "[ERR] Error occured, re-try %d" % retry_times
+            print >> sys.stderr, e
+
+        if has_send:
+            break
+
+    if not has_send:
+        print >> sys.stderr, "[ERR] Some error occured when sending email."
+
+if __name__ == '__main__':
+    main()

py/sa/obj.pkl

Binary file modified.
+import re
+import os
+
+pat = re.compile(r'(\d{4})_(.+)')
+filenames = [f for f  in os.listdir('.') if pat.search(f)]
+for fn in filenames:
+    m = pat.search(fn)
+    _d = m.group(1)
+    _fn = m.group(2)
+    os.makedirs(_d)
+    os.rename(fn , os.path.join(_d, _fn))
+
+    print _d, _fn
+
+
+def get_size(s):
+    _u = 1024
+    KB = _u
+    MB = _u ** 2
+    GB = _u ** 3
+
+    size_map = {
+        'kb': KB,
+        'mb': MB,
+        'gb': GB}
+    s = s.lower().strip()
+    size = 0.0
+    for k, v in size_map.iteritems():
+        if k in s:
+            s = s.strip(k).strip()
+            try:
+                size = float(s) * v
+            except ValueError:
+                print 'error string', s
+                break
+    size = int(size)
+    return size
+
+
+print get_size('100 MB')
+print get_size('20 KB')
+print get_size('1 GB')
+# -*- encoding:utf-8 -*-
+import sys
+import os
+from hashlib import sha1
+
+def mkdir(p):
+    if not os.path.exists(p):
+        print 'mkdir', p
+
+        os.makedirs(p)
+
+def slice(li, step):
+    num = len(li)
+    _li = range(0, num, step)
+
+    for index, elem in enumerate(_li):
+        if index == (len(_li) - 1):
+            end = None
+        else:
+            end = _li[index + 1]
+        yield li[elem:end]
+
+def hash(s):
+    hx = sha1(s).hexdigest()
+    return (hx[0:2], hx[2:4])
+
+def main():
+    path = sys.argv[1]
+    dest = os.path.dirname(path)
+
+    for root, dirs, files in os.walk(path):
+        for fn in files:
+            new_path = '/'.join(hash(fn))
+            fp = os.path.join(root, fn)
+            new_fp = os.path.join(dest, new_path, fn)
+            mkdir(os.path.dirname(new_fp))
+            print 'mv %s -> %s' % (fp, new_fp)
+            os.rename(fp, new_fp)
+
+if __name__ == '__main__':
+    main()
+import mmap
+import os
+
+#with open('test.py', 'rb') as fb:
+    ## fd = os.open('test.py', os.O_RDONLY)
+    #fd = fb.fileno()
+    #m = mmap.mmap(fd, 0)
+    #print m.readline()
+    #m.close()
+
+m = mmap.mmap(-1, 13)
+m.write('hello world!')
+
+pid = os.fork()
+
+if pid == 0: # In a child process
+    m.seek(0)
+    print m.readline()
+    m.close()
+