Snippets

xchandan md5_large.py

Created by xchandan
import os
import sys
import hashlib


def md5_for_file(f, block_size=2**20):
    fd = f.fileno()
    md5 = hashlib.md5()
    while True:
        data = f.read(block_size)
        if not data:
            break
        #os.posix_fadvise(fd, f.tell(), len(data), os.POSIX_FADV_DONTNEED)
        os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED)
        md5.update(data)
    return md5.hexdigest()

f = open(sys.argv[1], 'rb')
print(md5_for_file(f), sys.argv[1])

Comments (0)

HTTPS SSH

You can clone a snippet to your computer for local editing. Learn more.