Source

pinyomi / pixel / pixelSimilarity.py

Full commit
# -*- coding: utf-8 -*-
#----------------------------------------------------------------------------#
# pixelSimilarity.py
# Lars Yencken <lars.yencken@gmail.com>
# vim: ts=4 sw=4 sts=4 et tw=78:
# $Id: pixelSimilarity.py 376 2007-04-17 06:15:56Z lars $
#
#----------------------------------------------------------------------------#

""" This module contains methods to calculate pixel-based similarity scores
    between kanji.
"""

#----------------------------------------------------------------------------#

from os import path
import os
import Image
import ImageChops

from cjktools.stats import mean
from www import settings

from renderFont import kanjiToPng
from pixel.common import characterPath

#----------------------------------------------------------------------------#

# The directory where the rendered fonts are found.
_fontDir = path.join(settings.DATA_DIR, 'kanji-family')
#_fontDir = path.join(_thisDir)

_thisDir = path.join(path.dirname(__file__))
_pngLib = path.join(_thisDir, 'comparePng.so')

#----------------------------------------------------------------------------#

from comparePng import PngImage

#----------------------------------------------------------------------------#
# PUBLIC METHODS
#----------------------------------------------------------------------------#

_pixelMu = 0.52408803251981151
_pixelSigma = 0.047952354264500792

def pixelQuantized(kanjiA, kanjiB, threshold=_pixelMu+1*_pixelSigma):
    """ This is identical to pixel similarity, but is quantized into a binary
        decision, using a threshold of mu + 1*sigma. This is useful in
        determining whether quantization has a significant part to play in 
        calculation of rank correlation.
    """
    realSimilarity = pixelSimilarity(kanjiA, kanjiB)
    return int(realSimilarity > threshold)

#----------------------------------------------------------------------------#

def pixelQuantized3Sig(kanjiA, kanjiB):
    """ The same as pixelQuantized, but the threshold is set to mu + 3*sigma,
        in order to limit to really similar pairs. On average this gives
        reports 1.3 highly similar pairs per kanji.
    """
    return pixelQuantized(kanjiA, kanjiB, _pixelMu+3*_pixelSigma)

#----------------------------------------------------------------------------#

def pixelSimilarityCpp(kanjiA, kanjiB, font=settings.PIXEL_DEFAULT_FONT):
    """ Calculates the straight pixel similarity using the comparePng
        extension.
    """
    imageA = _lazyLoadImage(kanjiA, font)
    imageB = _lazyLoadImage(kanjiB, font)

    return 1.0 - imageA.norm(imageB, 1.0)

#----------------------------------------------------------------------------#

def pixelSimilarityPy(kanjiA, kanjiB, font=settings.PIXEL_DEFAULT_FONT):
    """ Returns a number between 0 and 1 determining the similarity predicted
        by OCR methods, where 1 is identical, and 0 is completely different.
    """
    imageA = Image.open(_imagePath(kanjiA, font)).convert('L')
    imageB = Image.open(_imagePath(kanjiB, font)).convert('L')

    diffImage = ImageChops.difference(imageA, imageB)

    difference = mean(list(diffImage.getdata()))/255.0

    similarity = 1.0 - difference

    return similarity

pixelSimilarity = pixelSimilarityCpp

#----------------------------------------------------------------------------#

def crossFontSimilarity(kanjiA, fontA, kanjiB, fontB):
    """ Calculates similarity across fonts.
    """
    imageA = _fetchImage(kanjiA, fontA).convert('L')
    imageB = _fetchImage(kanjiB, fontB).convert('L')

    diffImage = ImageChops.difference(imageA, imageB)

    difference = mean(list(diffImage.getData()))/255.0

    similarity = 1.0 - difference

    return similarity

#----------------------------------------------------------------------------#

def meanPixelSimilarity(kanjiA, kanjiB):
    """ Returns the similarity between the two kanji, but averaging over the
        similarity given by a number of fonts.
    """
    global _availableFonts

    if _availableFonts is None:
        _availableFonts =_detectFonts()

    values = []
    for font in _availableFonts:
        values.append(pixelSimilarity(kanjiA, kanjiB, font))

    return mean(values)

#----------------------------------------------------------------------------#

def getFonts():
    """ Gets a list of usable fonts.
    """
    global _availableFonts

    if _availableFonts is None:
        _availableFonts = _detectFonts()
    
    return _availableFonts

#----------------------------------------------------------------------------#

class FontNotRenderedError(Exception):
    """ This error should be thrown when code attempts to use a font which
        hasn't been rendered yet.
    """
    pass

#----------------------------------------------------------------------------#
# PRIVATE METHODS
#----------------------------------------------------------------------------#

_imageCache = {}
_accessHistory = []
_lastFont = None

def _lazyLoadImage(kanji, font, maxElements=10):
    """ Loads an image from the image cache lazily, only creating a new
        instance if one doesn't already exist. Warning: memory can grow
        unboundedly.
    """
    global _imageCache, _accessHistory, _lastFont
    
    if font != _lastFont:
        # Clear the cache
        _imageCache = {}
        _accessHistory = []
        _lastAccess = 0
        _lastFont = font

    image = _imageCache.get(kanji)

    if image is not None:
        # Cache hit!
        _accessHistory.append(_accessHistory.pop(_accessHistory.index(kanji)))

    else:
        # Cache miss!
        filename = _imagePath(kanji, font)
        if not path.exists(filename):
            raise FontNotRenderedError, '%s %s' % (filename, kanji)
            # XXX we could of course try to render anything missing
#            if not path.isdir(path.dirname(filename)):
#                os.makedirs(path.dirname(filename))
#            kanjiToPng(kanji, filename, font, size=60)

        assert path.exists(filename)

        image = PngImage(filename)

        if len(_imageCache) == maxElements:
            expiredKanji = _accessHistory.pop(0)
            del _imageCache[expiredKanji]

        _accessHistory.append(kanji)
        _imageCache[kanji] = image

    return image

#----------------------------------------------------------------------------#

_availableFonts = None

#----------------------------------------------------------------------------#

def _detectFonts():
    """ Detect the available fonts.
    """
    global _fontDir
    detectedFonts = os.listdir(_fontDir)
    return detectedFonts

#----------------------------------------------------------------------------#

def _imagePath(kanji, font):
    """ Determine the full path of the image for the given kanji, using the
        specified font.

        @param kanji: The kanji whose image to fetch.
        @param font: The font to use.
    """
    # Determine where the font's image collection is.
    global _fontDir
    fullFont = path.join(_fontDir, font)
    if not path.isdir(fullFont):
        raise FontNotRenderedError, "No such rendered font %s" % `fullFont`

    # Construct the fully qualified name.
    imagePath = path.join(fullFont, characterPath(kanji))

    return imagePath

#----------------------------------------------------------------------------#