Commits

Scott Farquhar  committed 1b1774e

Initial Commit of generic lucene stats package.

  • Participants

Comments (0)

Files changed (29)

+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>com.atlassian.pom</groupId>
+        <artifactId>atlassian-public-pom</artifactId>
+        <version>28</version>
+    </parent>
+
+    <groupId>com.atlassian.lucene.stats</groupId>
+    <artifactId>lucenestats</artifactId>
+    <version>1.0-SNAPSHOT</version>
+
+    <!--<scm>-->
+    <!--<connection>scm:git:git://git.cedarsoft.com/com.atlassian.lucene.stats.lucenestats</connection>-->
+    <!--<developerConnection>scm:git:ssh://git.cedarsoft.com/home/git/com.atlassian.lucene.stats.lucenestats</developerConnection>-->
+    <!--</scm>-->
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.lucene</groupId>
+            <artifactId>lucene-core</artifactId>
+            <version>2.9.4</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.collections</groupId>
+            <artifactId>google-collections</artifactId>
+            <version>1.0</version>
+        </dependency>
+
+
+        <!--Test Dependencies-->
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+            <version>4.8.1</version>
+        </dependency>
+    </dependencies>
+
+</project>

File src/main/java/com/atlassian/lucene/stats/CachingStatisticsMapper.java

+package com.atlassian.lucene.stats;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A statisticsMapper that caches the return value from {@link #getValueFromLuceneField(String)} in an internal
+ * cache.
+ * <p/>
+ * As the cache is not bounded, this object should not be stored for longer than a request
+ */
+public class CachingStatisticsMapper extends StatisticsMapperWrapper
+{
+    private final Map valuesCache = new HashMap();
+
+    public CachingStatisticsMapper(StatisticsMapper statisticsMapper)
+    {
+        super(statisticsMapper);
+    }
+
+    /**
+     * As lookups may be expensive, we cache the String->Object values in a cache
+     */
+    public Object getValueFromLuceneField(String documentValue)
+    {
+        Object value = valuesCache.get(documentValue);
+        if (value == null)
+        {
+            value = super.getValueFromLuceneField(documentValue);
+            valuesCache.put(documentValue, value);
+        }
+        return value;
+    }
+}

File src/main/java/com/atlassian/lucene/stats/DelegatingComparator.java

+package com.atlassian.lucene.stats;
+
+import java.util.Comparator;
+
+public class DelegatingComparator implements Comparator
+{
+    private final Comparator comparator1;
+    private final Comparator comparator2;
+
+    /**
+     * Constructs an instace of this comparator setting the first
+     * (top-priority) comparator and second (lower-priority) comparator.
+     * @param comparator1 comparator
+     * @param comparator2 comparator
+     */
+    public DelegatingComparator(Comparator comparator1, Comparator comparator2)
+    {
+        this.comparator1 = comparator1;
+        this.comparator2 = comparator2;
+    }
+
+    /**
+     * Compares two given objects. Uses {@link #comparator1} first and returns
+     * the result of comparison if not 0. In case of 0, it continues and
+     * returns the result of comparison using {@link #comparator2}.
+     * @param o1 object to compare
+     * @param o2 object to compare
+     * @return result of comparison
+     */
+    public int compare(Object o1, Object o2)
+    {
+        int result = comparator1.compare(o1, o2);
+        if (result != 0)
+        {
+            return result;
+        }
+        else
+        {
+            return comparator2.compare(o1, o2);
+        }
+    }
+}

File src/main/java/com/atlassian/lucene/stats/JiraLuceneFieldFinder.java

+package com.atlassian.lucene.stats;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermDocs;
+import org.apache.lucene.index.TermEnum;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+
+/**
+ * This used to be a cache of values but it was found that it consumed a hell of a lot of memory for no benefit
+ * (JRA-10111). So the cache.put was never called.
+ * <p/>
+ * This has been refactored into a "finder" of terms values for fields within documents.
+ */
+public class JiraLuceneFieldFinder
+{
+    private static final JiraLuceneFieldFinder FIELD_FINDER = new JiraLuceneFieldFinder();
+
+    public static JiraLuceneFieldFinder getInstance()
+    {
+        return FIELD_FINDER;
+    }
+
+    /**
+     * This is used to retrieve values from the Lucence index.  It returns an array that is the same size as the number
+     * of documents in the reader and will have all null values if the field is not present, otherwise it has the values
+     * of the field within the document.
+     *
+     * @param reader the Lucence index reader
+     * @param field the name of the field to find
+     * @param luceneFieldSorter Converts fields into values that are used if there are multiple values stored
+     * @return an non null array of values, which may contain null values.
+     * @throws IOException if things dont play out well.
+     */
+    public Object[] getCustom(IndexReader reader, final String field, LuceneFieldSorter luceneFieldSorter)
+            throws IOException
+    {
+        String internedField = field.intern();
+        final Object[] retArray = new Object[reader.maxDoc()];
+        if (retArray.length > 0)
+        {
+            TermDocs termDocs = reader.termDocs();
+            TermEnum termEnum = reader.terms(new Term(internedField, ""));
+            try
+            {
+                // if we dont have a term in any of the documents
+                // then an array of null values is what we should return
+                if (termEnum.term() == null)
+                {
+                    return retArray;
+                }
+                do
+                {
+                    Term term = termEnum.term();
+                    // Because Lucence interns fields for us this is a bit quicker
+                    //noinspection StringEquality
+                    if (term.field() != internedField)
+                    {
+                        // if the next term is not our field then none of those
+                        // terms are present in the set of documents and hence
+                        // an array of null values is what we should return
+                        break;
+                    }
+                    Object termval = luceneFieldSorter.getValueFromLuceneField(term.text());
+                    termDocs.seek(termEnum);
+                    while (termDocs.next())
+                    {
+                        Object currentValue = retArray[termDocs.doc()];
+                        //only replace the value if it is earlier than the current value
+                        Comparator comparator = luceneFieldSorter.getComparator();
+                        //noinspection unchecked
+                        if (currentValue == null || comparator.compare(termval, currentValue) < 1)
+                        {
+                            retArray[termDocs.doc()] = termval;
+                        }
+                    }
+                }
+                while (termEnum.next());
+            }
+            finally
+            {
+                termDocs.close();
+                termEnum.close();
+            }
+        }
+        return retArray;
+    }
+
+    /**
+     * For each document in the index, it returns an array of string collections for each matching term.
+     *
+     * @param reader the index to read
+     * @param field the field to check the documents for
+     * @return an array of string collections for each term for each document
+     * @throws IOException if things dont play out well.
+     */
+    public Collection<String>[] getMatches(IndexReader reader, final String field) throws IOException
+    {
+        String internedField = field.intern();
+        @SuppressWarnings ( { "unchecked" })
+        final Collection<String>[] docToTerms = new Collection[reader.maxDoc()];
+        if (docToTerms.length > 0)
+        {
+            TermDocs termDocs = reader.termDocs();
+            TermEnum termEnum = reader.terms(new Term(internedField, ""));
+            try
+            {
+                if (termEnum.term() == null)
+                {
+                    throw new RuntimeException("no terms in field " + field);
+                }
+                do
+                {
+                    Term term = termEnum.term();
+                    // Because Lucene interns fields for us this is a bit quicker
+                    //noinspection StringEquality
+                    if (term.field() != internedField)
+                    {
+                        break;
+                    }
+                    String termval = term.text();
+                    termDocs.seek(termEnum);
+                    while (termDocs.next())
+                    {
+                        Collection<String> currentValue = docToTerms[termDocs.doc()];
+                        if (currentValue == null)
+                        {
+                            currentValue = new ArrayList<String>();
+                        }
+
+                        currentValue.add(termval);
+                        docToTerms[termDocs.doc()] = currentValue;
+                    }
+                }
+                while (termEnum.next());
+            }
+            finally
+            {
+                termDocs.close();
+                termEnum.close();
+            }
+        }
+        return docToTerms;
+    }
+}

File src/main/java/com/atlassian/lucene/stats/LongFieldStatisticsMapper.java

+package com.atlassian.lucene.stats;
+
+import java.util.Comparator;
+
+public class LongFieldStatisticsMapper implements StatisticsMapper {
+
+    private final String documentConstant;
+
+    public LongFieldStatisticsMapper(String documentConstant) {
+        this.documentConstant = documentConstant;
+    }
+
+    public String getDocumentConstant() {
+        return documentConstant;
+    }
+
+    public Object getValueFromLuceneField(String documentValue) {
+        if ("-1".equals(documentValue))
+            return null;
+        else
+            return new Long(documentValue);
+    }
+
+    public Comparator getComparator() {
+        return new Comparator<Long>() {
+            @Override
+            public int compare(final Long o1, final Long o2) {
+                if (o1.equals(o2)) {
+                    return 0;
+                }
+                if (o2 == null) {
+                    return 1;
+                }
+                return (o1).compareTo(o2);
+
+            }
+        };
+    }
+
+    public boolean isValidValue(Object value) {
+        return true;
+    }
+
+    @Override
+    public boolean equals(final Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (!(o instanceof LongFieldStatisticsMapper)) {
+            return false;
+        }
+
+        final LongFieldStatisticsMapper that = (LongFieldStatisticsMapper) o;
+
+        return !(documentConstant != null ? !documentConstant.equals(that.documentConstant) : that.documentConstant != null);
+
+    }
+
+    public int hashCode() {
+        return (getDocumentConstant() != null ? getDocumentConstant().hashCode() : 0);
+    }
+}
+

File src/main/java/com/atlassian/lucene/stats/LuceneFieldSorter.java

+package com.atlassian.lucene.stats;
+
+import java.util.Comparator;
+
+/**
+ * Implementations of this interface are used to sort Lucene search results of Issue Documents.
+ * <p/>
+ * <strong>NOTE</strong>: instances of this interface are <strong>cached</strong> by Lucene and are
+ * <strong>REUSED</strong> to sort multiple Lucene search results. The Comparator returned by the
+ * {@link #getComparator()} method could be used by Lucene from multiple threads at once.
+ * <p/>
+ * Therefore, the implementations of this interface <strong>MUST</strong> implement the {@link Object#equals(Object)}
+ * and {@link Object#hashCode()} methods correctly to ensure that Lucene can find the implementations of this class
+ * in its cache and reuse it, rather than make the cache grow indefinitely. (Unfortunately the Lucene cache is rather
+ * primitive at the moment, and is not bound).
+ * <p/>
+ * Also, ensure that the {@link Comparator} returned by the {@link #getComparator()} method is <strong>thread
+ * safe</strong>.
+ * <p/>
+ * As instances of this and the {@link Comparator} returned by this object are cached and reused by Lucene to sort
+ * multiple search results, the best thing to do is to ensure the implementations of this interface and the
+ * {@link Comparator} that is returned <strong>are immutable</strong> and that the {@link #equals(Object)} and
+ * {@link #hashCode()} methods respect the state of the object.
+ */
+public interface LuceneFieldSorter<T>
+{
+    /**
+     * @return the constant that this field is indexed with.
+     */
+    String getDocumentConstant();
+
+    /**
+     * Convert the lucene document field back to the object that you wish to use to display it.
+     * <p>
+     * eg. '1000' -> Version 1.
+     * <p>
+     * This should do the reverse of whatever you use to index the object into the document.
+     *
+     * @param   documentValue   The value of the field in the lucene index
+     * @return  The value that will be passed to the display
+     */
+    T getValueFromLuceneField(String documentValue);
+
+    /**
+     * A comparator that can be used to order objects returned by {@link #getValueFromLuceneField(String)}.
+     * <p/>
+     * The Comparator <strong>must</strong> be reentrant as it could be used by Lucene from multiple threads at once.
+     *
+     * @return  A comparator that compares objects and also handles nulls
+     */
+    Comparator<T> getComparator();
+
+    /**
+     * As this object is used as a key in a cache, this method <strong>must</strong> be provided and respect all internal state.
+     * <p/>
+     * See the class javadoc entry for more details.
+     */
+    boolean equals(Object obj);
+
+    /**
+     * As this object is used as a key in a cache, this method <strong>must</strong> be provided and respect all internal state.
+     * <p/>
+     * See the class javadoc entry for more details.
+     */
+    int hashCode();
+}

File src/main/java/com/atlassian/lucene/stats/MappedSortComparator.java

+package com.atlassian.lucene.stats;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.ScoreDocComparator;
+import org.apache.lucene.search.SortComparatorSource;
+import org.apache.lucene.search.SortField;
+
+import java.io.IOException;
+import java.util.Comparator;
+
+/**
+ * This Sort Comparator reads through the terms dictionary in lucene, and builds up a list of ordered terms.  It then
+ * sorts the documents according to the order that they appear in the terms list.
+ * <p/>
+ * This approach, whilst very fast, does load the entire term dictionary into memory.  This could be a problem where there
+ * are a large number of terms (eg. text fields).
+ */
+public class MappedSortComparator implements SortComparatorSource
+{
+    private final LuceneFieldSorter sorter;
+
+    public MappedSortComparator(LuceneFieldSorter sorter)
+    {
+        this.sorter = sorter;
+    }
+
+    public ScoreDocComparator newComparator(final IndexReader reader, final String fieldname) throws IOException
+    {
+        return getComparatorUsingTerms(fieldname, reader);
+    }
+
+    private ScoreDocComparator getComparatorUsingTerms(final String fieldname, final IndexReader reader) throws IOException
+    {
+        final String field = fieldname.intern();
+        final Object[] cachedValues = getLuceneValues(field, reader);
+        final Comparator comparator = sorter.getComparator();
+
+        return new ScoreDocComparator()
+        {
+            public int compare(ScoreDoc i, ScoreDoc j)
+            {
+                Object value1 = cachedValues[i.doc];
+                Object value2 = cachedValues[j.doc];
+                if (value1 == null && value2 == null) // if they are both null, they are the same.  Fixes JRA-7003
+                {
+                    return 0;
+                }
+                else if (value1 == null)
+                {
+                    return 1;  //null is greater than any value (we want these at the end)
+                }
+                else if (value2 == null)
+                {
+                    return -1; // any value is less than null (we want null at the end)
+                }
+                else
+                {
+                    return comparator.compare(value1, value2);
+                }
+            }
+
+            public Comparable sortValue(ScoreDoc i)
+            {
+                return null; //we won't be able to pull the values from the sort
+            }
+
+            public int sortType()
+            {
+                return SortField.CUSTOM;
+            }
+        };
+    }
+
+    /**
+     * This makes a call into the JiraLucenceFieldCache to retrieve values from the Lucence
+     * index.  It returns an array that is the same size as the number of documents in the reader
+     * andwill have all null values if the field is not present, otherwise it has the values
+     * of the field within the document.
+     * <p/>
+     * Broken out as package level for unit testing reasons.
+     *
+     * @param field  the name of the field to find
+     * @param reader the Lucence index reader
+     * @return an non null array of values, which may contain null values.
+     * @throws IOException if stuff goes wrong
+     */
+    Object[] getLuceneValues(final String field, final IndexReader reader) throws IOException
+    {
+        return JiraLuceneFieldFinder.getInstance().getCustom(reader, field, sorter);
+    }
+
+    /**
+     * Returns an object which, when sorted according by the comparator returned from  {@link LuceneFieldSorter#getComparator()} ,
+     * will order the Term values in the correct order.
+     * <p>For example, if the Terms contained integer values, this method
+     * would return <code>new Integer(termtext)</code>.  Note that this
+     * might not always be the most efficient implementation - for this
+     * particular example, a better implementation might be to make a
+     * ScoreDocLookupComparator that uses an internal lookup table of int.
+     *
+     * @param termtext The textual value of the term.
+     * @return An object representing <code>termtext</code> that can be sorted by {@link LuceneFieldSorter#getComparator()}
+     * @see Comparable
+     * @see ScoreDocComparator
+     */
+    public Object getComparable(String termtext)
+    {
+        return sorter.getValueFromLuceneField(termtext);
+    }
+
+    public Comparator getComparator()
+    {
+        return sorter.getComparator();
+    }
+
+    public boolean equals(Object o)
+    {
+        if (this == o)
+        {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass())
+        {
+            return false;
+        }
+
+        final MappedSortComparator that = (MappedSortComparator) o;
+
+        return (sorter == null ? that.sorter == null : sorter.equals(that.sorter));
+    }
+
+    public int hashCode()
+    {
+        return (sorter != null ? sorter.hashCode() : 0);
+    }
+}

File src/main/java/com/atlassian/lucene/stats/OneDimensionalObjectHitCollector.java

+package com.atlassian.lucene.stats;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.HitCollector;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ * A HitCollector that creates a doc -> object mapping.  This is useful for collecting documents where there are a
+ * limited number of terms.  The caching also ensures that if multiple searches sort on the same terms, the doc ->
+ * object mapping is maintained.
+ * <p/>
+ * This HitCollector can be quite memory intensive, however the cache is stored with a weak reference, so it will be
+ * garbage collected.
+ * <p/>
+ * This HitCollector differs from {@link OneDimensionalTermHitCollector} in that it performs the term -> object
+ * conversion here, rather than later.  This is more expensive, but useful for StatisticsMappers that perform some sort
+ * of runtime conversion / translation (eg a StatisticsMapper that groups dates by Month, or groups users by email
+ * domain name).
+ */
+public class OneDimensionalObjectHitCollector extends HitCollector
+{
+    private StatisticsMapper<Object> statisticsMapper;
+    private final Map<Object, Integer> result;
+    private Collection<String>[] docToTerms;
+
+    public OneDimensionalObjectHitCollector(StatisticsMapper<Object> statisticsMapper, Map<Object, Integer> result, IndexReader indexReader)
+    {
+        this.result = result;
+        this.statisticsMapper = statisticsMapper;
+        try
+        {
+            docToTerms = JiraLuceneFieldFinder.getInstance().getMatches(indexReader, statisticsMapper.getDocumentConstant());
+        }
+        catch (IOException e)
+        {
+            //ignore
+        }
+    }
+
+    public void collect(int i, float v)
+    {
+        adjustMapForValues(result, docToTerms[i]);
+    }
+
+    private void adjustMapForValues(Map<Object, Integer> map, Collection<String> terms)
+    {
+        if (terms == null)
+        {
+            return;
+        }
+        for (String term : terms)
+        {
+            Object object = statisticsMapper.getValueFromLuceneField(term);
+            Integer count = map.get(object);
+
+            if (count == null)
+            {
+                count = 0;
+            }
+            map.put(object, count + 1);
+        }
+    }
+
+}

File src/main/java/com/atlassian/lucene/stats/OneDimensionalTermHitCollector.java

+package com.atlassian.lucene.stats;
+
+import com.atlassian.lucene.stats.cache.ReaderCache;
+import com.atlassian.lucene.stats.util.Supplier;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.HitCollector;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ * A HitCollector that creates a doc -> term mapping.  This is useful for collecting documents where there are a
+ * limited number of terms.  The caching also ensures that if multiple searches sort on the same terms, the doc -> term
+ * mapping is maintained.
+ * <p/>
+ * This HitCollector can be quite memory intensive, however the cache is stored with a weak reference, so it will
+ * be garbage collected.
+ * //todo - fix this!!!  Use everything from TwoDimensionalTermHitCollector
+ */
+public class OneDimensionalTermHitCollector extends HitCollector {
+    private Collection<String>[] docToTerms;
+    private final Map<String, Integer> result;
+
+
+//    /**
+//     * Records the number of times the {@link #collect(int, float)} method was called. The method
+//     * should be called once for each issue.
+//     */
+//    private long hitCount = 0;
+
+    public OneDimensionalTermHitCollector(final String fieldId, final Map<String, Integer> result,
+                                          final IndexReader indexReader,
+                                          final ReaderCache readerCache) {
+        this.result = result;
+
+        this.docToTerms = readerCache.get(indexReader, fieldId, new Supplier<Collection<String>[]>() {
+            public Collection<String>[] get() {
+                try {
+                    return JiraLuceneFieldFinder.getInstance().getMatches(indexReader, fieldId);
+                } catch (IOException e) {
+                    throw new RuntimeIOException(e);
+                }
+            }
+        });
+    }
+
+
+    public void collect(int i, float v) {
+//        ++hitCount;
+        Collection<String> terms = docToTerms[i];
+        if (terms == null) {
+            incrementCount(null, result);
+        } else {
+            for (String term : terms) {
+                incrementCount(term, result);
+            }
+        }
+    }
+
+
+//    /**
+//     * Returns the number of times the {@link #collect(int, float)} method was called.
+//     * This should return the number of unique issues that was matched during a search.
+//     * @return number of times the {@link #collect(int, float)} method was called.
+//     */
+//    public long getHitCount()
+//    {
+//        return hitCount;
+//    }
+
+
+    private void incrementCount(final String key, final Map<String, Integer> map) {
+        Integer count = map.get(key);
+
+        if (count == null) {
+            count = 1;
+        } else {
+            count++;
+        }
+        map.put(key, count);
+    }
+
+}

File src/main/java/com/atlassian/lucene/stats/RuntimeIOException.java

+package com.atlassian.lucene.stats;
+
+import com.atlassian.lucene.stats.util.NotNull;
+
+import java.io.IOException;
+
+/**
+ * An IOException was encountered and the stupid programmer didn't know how to recover, so this got thrown instead.
+ */
+public class RuntimeIOException extends RuntimeException
+{
+    private static final long serialVersionUID = -8317205499816761123L;
+
+    public RuntimeIOException(final @NotNull String message, final @NotNull IOException cause)
+    {
+        super(message, cause);
+    }
+
+    public RuntimeIOException(final @NotNull IOException cause)
+    {
+        super(cause);
+    }
+}

File src/main/java/com/atlassian/lucene/stats/StatisticGatherer.java

+package com.atlassian.lucene.stats;
+
+import java.util.*;
+
+/**
+ * A simple way of calculating statistics
+ */
+public interface StatisticGatherer
+{
+    /**
+     * Returns current value. If null, the return value will be the 0.
+     *
+     * @param current  The current value.  If null, the return value will be the equivalent of 0
+     * @param newValue The value to add to the current
+     * @return The new value - guaranteed not to be null
+     */
+    public Number getValue(Number current, int newValue);
+
+    public static class Sum implements StatisticGatherer
+    {
+        public Number getValue(Number current, int newValue)
+        {
+            if (current == null)
+            {
+                current = newValue;
+            }
+            else
+            {
+                current = newValue + current.intValue();
+            }
+            return current;
+        }
+    }
+
+    public static class Mean implements StatisticGatherer
+    {
+        public Number getValue(Number current, int newValue)
+        {
+            MeanValue meanValue = (MeanValue) current;
+            if (meanValue == null)
+            {
+                meanValue = new MeanValue();
+            }
+            meanValue.addValue(newValue);
+            return meanValue;
+        }
+    }
+
+    public static class Median implements StatisticGatherer
+    {
+        public Number getValue(Number current, int newValue)
+        {
+            MedianValue medianValue = (MedianValue) current;
+            if (medianValue == null)
+            {
+                medianValue = new MedianValue();
+            }
+            medianValue.addValue(newValue);
+            return medianValue;
+        }
+    }
+
+    public static class CountUnique implements StatisticGatherer
+    {
+        public Number getValue(Number current, int newValue)
+        {
+            CountUniqueValue countUniqueValue = (CountUniqueValue) current;
+            if (countUniqueValue == null)
+            {
+                countUniqueValue = new CountUniqueValue();
+            }
+            countUniqueValue.addValue(newValue);
+            return countUniqueValue;
+        }
+    }
+
+    /**
+     * Using some trickiness - we extend Number so that we can call {@link #intValue()} without having
+     * to change all the client code
+     */
+    static class MeanValue extends Number
+    {
+        private int total;
+        private int count;
+
+        public void addValue(int value)
+        {
+            total += value;
+            count++;
+        }
+
+        public int intValue()
+        {
+            return total / count;
+        }
+
+        public long longValue()
+        {
+            return intValue();
+        }
+
+        public float floatValue()
+        {
+            return intValue();
+        }
+
+        public double doubleValue()
+        {
+            return intValue();
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.valueOf(intValue());
+        }
+    }
+
+    public static class MedianValue extends Number
+    {
+        private final List values = new ArrayList();
+
+        public void addValue(int value)
+        {
+            values.add(new Integer(value));
+        }
+
+        public int intValue()
+        {
+            Collections.sort(values, new Comparator()
+            {
+                public int compare(Object o, Object o1)
+                {
+                    if (o == null)
+                    {
+                        return -1; //null is smaller than anything
+                    }
+                    if (o1 == null)
+                    {
+                        return 1;
+                    }
+                    return ((Integer) o).compareTo((Integer) o1);
+
+                }
+            });
+            if (values.isEmpty())
+            {
+                return 0;
+            }
+            else if (values.size() % 2 == 0)
+            {
+                int i = (Integer) values.get((values.size()) / 2);
+                int j = (Integer) values.get(values.size() / 2 - 1);
+                return (i + j) / 2;
+            }
+            else
+            {
+                return (Integer) values.get(values.size() / 2);
+            }
+        }
+
+        public long longValue()
+        {
+            return intValue();
+        }
+
+        public float floatValue()
+        {
+            return intValue();
+        }
+
+        public double doubleValue()
+        {
+            return intValue();
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.valueOf(intValue());
+        }
+    }
+
+    static class CountUniqueValue extends Number
+    {
+        private final Set values = new HashSet();
+
+        public void addValue(int value)
+        {
+            values.add(value);
+        }
+
+        public int intValue()
+        {
+            return values.size();
+        }
+
+        public long longValue()
+        {
+            return intValue();
+        }
+
+        public float floatValue()
+        {
+            return intValue();
+        }
+
+        public double doubleValue()
+        {
+            return intValue();
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.valueOf(intValue());
+        }
+    }
+
+
+}

File src/main/java/com/atlassian/lucene/stats/StatisticsMapper.java

+package com.atlassian.lucene.stats;
+
+
+/**
+ * Allow mapping from Lucene indexes, back to the fields that they came from.
+ * <p/>
+ * Any 'field' that implements this is capable of having a statistic calculated from it.
+ *
+ * @see com.atlassian.jira.issue.statistics.FilterStatisticsValuesGenerator
+ */
+public interface StatisticsMapper<T> extends LuceneFieldSorter<T>
+{
+    /**
+     * Check whether this value is valid for this particular search.  This is useful if you do not wish to display
+     * all the values that are indexed (eg - only show released versions)
+     *
+     * @param value This is the same value that will be returned from {@link #getValueFromLuceneField(String)}
+     * @return true if this value is valid for this particular search
+     */
+    boolean isValidValue(T value);
+
+//    /**
+//     * Check if the field is always part of an issues data. This should only return false in the case of a
+//     * custom field where the value does not have to be set for each issue.
+//     *
+//     * @return true if this mapper will always be part of an issues data
+//     */
+//    boolean isFieldAlwaysPartOfAnIssue();
+
+    /**
+     * Get a suffix for the issue navigator, which allows for filtering on this value.
+     * <p/>
+     * eg. a project field would return a SearchRequest object who's getQueryString method will produce
+     * <code>pid=10240</code>
+     * <p/>
+     * Note that values returned from implementations should return values that are URLEncoded.
+     *
+     * @param value         This is the same value that will be returned from {@link #getValueFromLuceneField(String)}
+     * @param searchRequest is the search request that should be used as the base of the newly generated
+     *                      SearchRequest object. If this parameter is null then the return type will also be null.
+     * @return a SearchRequest object that will generate the correct issue navigator url to search
+     *         the correct statistics set, null otherwise.
+     * @see java.net.URLEncoder#encode(String)
+     */
+//    SearchRequest getSearchUrlSuffix(T value, SearchRequest searchRequest);
+}

File src/main/java/com/atlassian/lucene/stats/StatisticsMapperWrapper.java

+package com.atlassian.lucene.stats;
+
+import java.util.Comparator;
+
+/**
+ * Provides a convenient implementation of the StatisticsMapper interface
+ * that can be subclassed by developers wishing to adapt the request to a mapper.
+ * This class implements the Wrapper or Decorator pattern. Methods default to
+ * calling through to the wrapped statisticsMapper object.
+ */
+public class StatisticsMapperWrapper implements StatisticsMapper
+{
+    private final StatisticsMapper statisticsMapper;
+
+    public StatisticsMapperWrapper(StatisticsMapper statisticsMapper)
+    {
+        this.statisticsMapper = statisticsMapper;
+    }
+
+    public boolean isValidValue(Object value)
+    {
+        return statisticsMapper.isValidValue(value);
+    }
+
+//    public boolean isFieldAlwaysPartOfAnIssue()
+//    {
+//        return statisticsMapper.isFieldAlwaysPartOfAnIssue();
+//    }
+
+//    public SearchRequest getSearchUrlSuffix(Object value, SearchRequest searchRequest)
+//    {
+//        return statisticsMapper.getSearchUrlSuffix(value, searchRequest);
+//    }
+
+    public String getDocumentConstant()
+    {
+        return statisticsMapper.getDocumentConstant();
+    }
+
+    public Object getValueFromLuceneField(String documentValue)
+    {
+        return statisticsMapper.getValueFromLuceneField(documentValue);
+    }
+
+    public Comparator getComparator()
+    {
+        return statisticsMapper.getComparator();
+    }
+}

File src/main/java/com/atlassian/lucene/stats/TwoDimensionalStatsMap.java

+package com.atlassian.lucene.stats;
+
+import com.atlassian.lucene.stats.util.EasyList;
+
+import java.io.Serializable;
+import java.util.*;
+
+public class TwoDimensionalStatsMap
+{
+    public final static String TOTAL_ORDER = "total";
+    public final static String NATURAL_ORDER = "natural";
+    public final static String DESC = "desc";
+    public final static String ASC = "asc";
+
+    private final StatisticsMapper xAxisMapper;
+    private final StatisticsMapper yAxisMapper;
+    private final StatisticGatherer statisticGatherer;
+
+    private final Map xAxis;
+    private final Map<Object, Number> xAxisTotals;
+    private final Map<Object, Number> yAxisTotals;
+    private Number entireTotal;
+
+    public TwoDimensionalStatsMap(StatisticsMapper xAxisMapper, StatisticsMapper yAxisMapper, StatisticGatherer statisticGatherer)
+    {
+        this.xAxisMapper = new CachingStatisticsMapper(xAxisMapper);
+        this.yAxisMapper = new CachingStatisticsMapper(yAxisMapper);
+        this.statisticGatherer = statisticGatherer;
+        xAxis = new TreeMap(xAxisMapper.getComparator());
+        xAxisTotals = new HashMap<Object, Number>();
+        yAxisTotals = new HashMap<Object, Number>();
+    }
+
+    public TwoDimensionalStatsMap(StatisticsMapper xAxisMapper, StatisticsMapper yAxisMapper)
+    {
+        this(xAxisMapper, yAxisMapper, new StatisticGatherer.Sum());
+    }
+
+    /**
+     * This method will increment the unique totals count for the provided
+     * xKey.
+     *
+     * @param xValue identifies the xValue we are keying on, null is valid.
+     * @param i      the amount to increment the total by, usually 1.
+     */
+    private void addToXTotal(Object xValue, int i)
+    {
+        Number total = xAxisTotals.get(xValue);
+        xAxisTotals.put(xValue, statisticGatherer.getValue(total, i));
+    }
+
+    /**
+     * This method will increment the unique totals count for the y row identified by yKey.
+     *
+     * @param yValue identifies the yValue we are keying on, null is valid.
+     * @param i      the amount to increment the total by, usually 1.
+     */
+    private void addToYTotal(Object yValue, int i)
+    {
+        Number total = yAxisTotals.get(yValue);
+        yAxisTotals.put(yValue, statisticGatherer.getValue(total, i));
+    }
+
+    /**
+     * Increments the total count of unique issues added to this StatsMap.
+     *
+     * @param i the amount to increment the total by, usually 1.
+     */
+    private void addToEntireTotal(int i)
+    {
+        entireTotal = statisticGatherer.getValue(entireTotal, i);
+    }
+
+    // As this is used for testing - it is package private
+    void addValue(Object xValue, Object yValue, int i)
+    {
+        Map yValues = (Map) xAxis.get(xValue);
+        if (yValues == null)
+        {
+            yValues = new TreeMap(yAxisMapper.getComparator());
+            xAxis.put(xValue, yValues);
+        }
+
+        Number existingValue = (Number) yValues.get(yValue);
+        yValues.put(yValue, statisticGatherer.getValue(existingValue, i));
+    }
+
+
+    public Collection getXAxis()
+    {
+        final Set xVals = new TreeSet(xAxisMapper.getComparator());
+        if (xAxis.keySet() != null)
+        {
+            xVals.addAll(xAxis.keySet());
+        }
+        return xVals;
+    }
+
+    public Collection getYAxis()
+    {
+        return getYAxis(NATURAL_ORDER, ASC);
+    }
+
+    public Collection getYAxis(String orderBy, String direction)
+    {
+        Comparator comp;
+
+        if (orderBy != null && orderBy.equals(TOTAL_ORDER))
+        {
+            // Compare by total
+            comp = new Comparator()
+            {
+
+                public int compare(Object o1, Object o2)
+                {
+                    Long o1Long = (long) getYAxisUniqueTotal(o1);
+                    Long o2Long = (long) getYAxisUniqueTotal(o2);
+                    return o1Long.compareTo(o2Long);
+                }
+            };
+
+            // Only reverse total Comaparator, not field Comparator
+            if (direction != null && direction.equals(DESC))
+            {
+                comp = Collections.reverseOrder(comp);
+            }
+
+            // If totals are equal, delagate back to field comparator
+            comp = new DelegatingComparator(comp, yAxisMapper.getComparator());
+        }
+        else
+        {
+            comp = yAxisMapper.getComparator();
+            if (direction != null && direction.equals(DESC))
+            {
+                comp = Collections.reverseOrder(comp);
+            }
+        }
+
+        return getYAxis(comp);
+    }
+
+    public Collection getYAxis(Comparator comp)
+    {
+        Set yAxisKeys = new TreeSet(comp);
+
+        for (Iterator iterator = xAxis.values().iterator(); iterator.hasNext();)
+        {
+            Map yAxisValues = (Map) iterator.next();
+            yAxisKeys.addAll(yAxisValues.keySet());
+        }
+        return yAxisKeys;
+    }
+
+    public int getCoordinate(String xAxis, String yAxis)
+    {
+        Map yValues = (Map) this.xAxis.get(xAxis);
+        if (yValues == null)
+        {
+            return 0;
+        }
+
+        Number value = (Number) yValues.get(yAxis);
+        return value == null ? 0 : value.intValue();
+    }
+
+    public StatisticsMapper getyAxisMapper()
+    {
+        return yAxisMapper;
+    }
+
+    public StatisticsMapper getxAxisMapper()
+    {
+        return xAxisMapper;
+    }
+
+    /**
+     * Returns the value of unique issues contained in the column identified by xAxis.
+     *
+     * @param xAxis identifies the column who's total is requested, null is valid.
+     * @return number of unique issues for the identified column.
+     */
+    public int getXAxisUniqueTotal(Serializable xAxis)
+    {
+        Number xTotal = xAxisTotals.get(xAxis);
+        return xTotal != null ? xTotal.intValue() : 0;
+    }
+
+    /**
+     * Returns the value of unique issues contained in the column identified by xAxis.
+     *
+     * @param yAxis identifies the row who's total is requested, null is valid.
+     * @return number of unique issues for the identified row.
+     */
+    public int getYAxisUniqueTotal(Object yAxis)
+    {
+        Number yTotal = yAxisTotals.get(yAxis);
+        return yTotal != null ? yTotal.intValue() : 0;
+    }
+
+    /**
+     * Returns the value of all unique issues identified within this StatsMap.
+     *
+     * @return number of unique issues identified within this StatsMap.
+     */
+    public long getUniqueTotal()
+    {
+        return entireTotal != null ? entireTotal.intValue() : 0;
+    }
+
+
+    public void adjustMapForValues(Collection xAxisValues, Collection yAxisValues, int value)
+    {
+        // if one axis is null, we still need something to iterate over to get the other axis' values
+        if (xAxisValues == null)
+        {
+            xAxisValues = EasyList.build((Object) null);
+        }
+        if (yAxisValues == null)
+        {
+            yAxisValues = EasyList.build((Object) null);
+        }
+
+        xAxisValues = transformAndRemoveInvalid(xAxisValues, xAxisMapper);
+        yAxisValues = transformAndRemoveInvalid(yAxisValues, yAxisMapper);
+
+        for (Object xvalue : xAxisValues) {
+            addToXTotal(xvalue, value);
+            for (Iterator iterator1 = yAxisValues.iterator(); iterator1.hasNext();) {
+                Object yvalue = iterator1.next();
+                addValue(xvalue, yvalue, value);
+
+            }
+        }
+        // We have to iterate over the y values alone so we don't mess up the totals
+        for (Object yvalue : yAxisValues) {
+            addToYTotal(yvalue, value);
+        }
+
+        // Always log one hit per unique issue.
+        addToEntireTotal(value);
+    }
+
+
+    /**
+     * Transform values in the collection from Strings to Objects,
+     * using {@link com.atlassian.jira.issue.search.LuceneFieldSorter#getValueFromLuceneField(String)}
+     *
+     * @param values A Collection of Strings, obtained from the Lucene Index
+     * @param mapper A statsMapper used to convert to objects
+     * @return a collection of transformed objects, never null
+     */
+    private static Collection transformAndRemoveInvalid(Collection values, StatisticsMapper mapper)
+    {
+        Collection output = new ArrayList();
+        for (Iterator iterator = values.iterator(); iterator.hasNext();)
+        {
+            String key = (String) iterator.next();
+            final Object value;
+            if (key != null)
+            {
+                value = mapper.getValueFromLuceneField(key);
+            }
+            else
+            {
+                value = null;
+            }
+
+            //only valid values should be added to the map
+            if (mapper.isValidValue(value))
+            {
+                output.add(value);
+            }
+        }
+        return output;
+    }
+
+
+}

File src/main/java/com/atlassian/lucene/stats/TwoDimensionalTermHitCollector.java

+package com.atlassian.lucene.stats;
+
+import com.atlassian.lucene.stats.cache.ReaderCache;
+import com.atlassian.lucene.stats.util.Supplier;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.HitCollector;
+
+import java.io.IOException;
+import java.util.Collection;
+
+/**
+ * A HitCollector that creates a doc -> term mapping.  This is useful for collecting documents where there are a
+ * limited number of terms.  The caching also ensures that if multiple searches sort on the same terms, the doc -> term
+ * mapping is maintained.
+ * <p/>
+ * This HitCollector can be quite memory intensive, however the cache is stored with a weak reference, so it will
+ * be garbage collected.
+ *
+ * @since v3.11
+ */
+public class TwoDimensionalTermHitCollector extends HitCollector
+{
+    private final TwoDimensionalStatsMap statsMap;
+    private final ReaderCache readerCache;
+    private final LuceneFieldSorter aggregateField;
+    private final Collection[] docToXTerms;
+    private final Collection[] docToYTerms;
+    private final Collection[] docToValueTerms;
+
+
+    public TwoDimensionalTermHitCollector(final TwoDimensionalStatsMap statsMap, final IndexReader indexReader,
+            final ReaderCache readerCache)
+    {
+        this(statsMap, indexReader, readerCache, null);
+    }
+
+    /**
+     * Update a statsMap, using the values from the <code>aggregateField</code>.  Example, you can sum the votes.
+     *
+     * @param statsMap       stats map
+     * @param indexReader    index reader
+     * @param readerCache used to cache stats values at the reader level
+     * @param aggregateField lucene field sorter.  Can be null if you just want to use Sum
+     */
+    public TwoDimensionalTermHitCollector(final TwoDimensionalStatsMap statsMap, final IndexReader indexReader,
+            final ReaderCache readerCache,
+            final LuceneFieldSorter aggregateField)
+    {
+        this.statsMap = statsMap;
+        this.readerCache = readerCache;
+        this.aggregateField = aggregateField;
+        this.docToXTerms = getDocToXTerms(indexReader, statsMap);
+        this.docToYTerms = getDocToYTerms(indexReader, statsMap);
+        // this will be null if there we are not aggregating by a field
+        this.docToValueTerms = getDocToValueTerms(aggregateField, indexReader);
+    }
+
+    public void collect(int i, float v)
+    {
+        adjustForValues(i);
+    }
+
+    private void adjustForValues(int docId)
+    {
+        final Collection xValues = docToXTerms[docId];
+        final Collection yValues = docToYTerms[docId];
+        int incrementValue = (docToValueTerms == null) ? 1 : getFieldValue(docId);
+            statsMap.adjustMapForValues(xValues, yValues, incrementValue);
+    }
+
+    private int getFieldValue(int i)
+    {
+        final Number value = getValue(docToValueTerms[i]);
+        return value != null ? value.intValue() : 0;
+    }
+
+    private Number getValue(Collection values)
+    {
+        if (values == null || values.isEmpty())
+        {
+            return null;
+        }
+        else if (values.size() > 1)
+        {
+            throw new IllegalArgumentException("More than one value stored for statistic \"" + values.toString() + "\".");
+        }
+        else
+        {
+            Object o = aggregateField.getValueFromLuceneField((String) values.iterator().next());
+
+            if (o == null)
+                return 0;
+
+            if (o instanceof Number)
+            {
+                return (Number) o;
+            }
+            else
+            {
+                throw new IllegalArgumentException("Value returned for statistic from " + aggregateField.getClass().getName() + " \"" + (o.getClass().getName()) + "\".  Expected \"java.lang.Number\"");
+            }
+        }
+    }
+
+    private Collection[] getDocToXTerms(IndexReader indexReader, TwoDimensionalStatsMap statsMap)
+    {
+        return getDocToValueTerms(statsMap.getxAxisMapper().getDocumentConstant(), indexReader);
+    }
+
+    private Collection[] getDocToYTerms(IndexReader indexReader, TwoDimensionalStatsMap statsMap)
+    {
+        return getDocToValueTerms(statsMap.getyAxisMapper().getDocumentConstant(), indexReader);
+    }
+
+    private Collection[] getDocToValueTerms(LuceneFieldSorter aggregateField, IndexReader indexReader)
+    {
+        if (aggregateField != null)
+        {
+            return getDocToValueTerms(aggregateField.getDocumentConstant(), indexReader);
+        }
+        return null;
+    }
+
+    private Collection<String>[] getDocToValueTerms(final String documentConstant, final IndexReader indexReader)
+    {
+        return readerCache.get(indexReader, documentConstant, new Supplier<Collection<String>[]>()
+        {
+            public Collection<String>[] get()
+            {
+                try
+                {
+                    return JiraLuceneFieldFinder.getInstance().getMatches(indexReader, documentConstant);
+                }
+                catch (IOException e)
+                {
+                    throw new RuntimeIOException(e);
+                }
+            }
+        });
+    }
+
+
+}

File src/main/java/com/atlassian/lucene/stats/cache/CompositeKeyCache.java

+package com.atlassian.lucene.stats.cache;
+
+import com.atlassian.lucene.stats.util.Function;
+import com.atlassian.lucene.stats.util.NotNull;
+import com.atlassian.lucene.stats.util.Supplier;
+import com.google.common.collect.MapMaker;
+import org.apache.lucene.index.IndexReader;
+
+import java.util.Map;
+
+import static com.atlassian.lucene.stats.util.Assertions.notNull;
+
+/**
+ * Cache of (R, S) -> T. Designed to be used for general mappings of things to a field in an index.
+ * <p>
+ * <strong>Usage:</strong>
+ * <pre>
+ * CompositeKeyCache&lt;IndexReader, String, Collection&lt;String&gt;[]&gt; cache = CompositeKeyCache.createWeakFirstKeySoftValueCache();
+ * cache.get(reader, fieldName,
+ *     new Supplier&lt;Collection&lt;String&gt;[]&gt;()
+ *     {
+ *         public Collection&lt;String&gt;[] get()
+ *         {
+ *             return doStuff(reader, fieldName);
+ *         }
+ *     }
+ * );
+ * </pre>
+ * @param <R> the first key, usually a string. Must be a type that can be used as a map key (ie. immutable with correct equals/hashcode).
+ * @param <S> the field name, usually a string. Must be a type that can be used as a map key (ie. immutable with correct equals/hashcode).
+ * @param <T> the result thing.
+ */
+public class CompositeKeyCache<R, S, T>
+{
+    /**
+     *  This cache caches the first (R) reference weakly, the second (S) reference strongly and the
+     * value (T) reference softly. This is specifically designed for use with Lucene where the
+     * {@link IndexReader} is being recycled regularly (the R) and the terms (T) may be softly referenced.
+     *
+     * @param <R> the first key type
+     * @param <S> the second key type
+     * @param <T> the value type
+     * @return a cache with weak references to the first key and soft references to the value.
+     */
+    public static <R, S, T> CompositeKeyCache<R, S, T> createWeakFirstKeySoftValueCache()
+    {
+        return new CompositeKeyCache<R, S, T>();
+    }
+
+    private final Cache cache = new Cache();
+
+    CompositeKeyCache()
+    {}
+
+    /**
+     * Get the thing mapped to this key for the specified reader.
+     *
+     * @param one the first one
+     * @param two the second one
+     * @param supplier to generate the value if not already there, only called if not already cached.
+     * @return the cached value
+     */
+    public T get(@NotNull final R one, @NotNull final S two, final Supplier<T> supplier)
+    {
+        return cache.get(one).get(new Key<S, T>(two, supplier));
+    }
+
+    /**
+     * Weakly map the R to a map of the second reference to the value.
+     */
+    private class Cache implements Function<R, Function<Key<S, T>, T>>
+    {
+        private final Map<R, Function<Key<S, T>, T>> map = new MapMaker().weakKeys().makeComputingMap(
+        // create the thing that holds the actual values
+            new com.google.common.base.Function<R, Function<Key<S, T>, T>>()
+            {
+                public Function<Key<S, T>, T> apply(final R from)
+                {
+                    return new ValueMap<S, T>();
+                }
+            });
+
+        public Function<Key<S, T>, T> get(final R from)
+        {
+            return map.get(from);
+        }
+    }
+
+    /**
+     * Hold the actual value, mapped from a Key.
+     */
+    static class ValueMap<S, T> implements Function<Key<S, T>, T>
+    {
+        private final Map<Key<S, T>, T> map = new MapMaker().softValues().makeComputingMap(
+        // create the thing that holds the actual value, just an adapter from our function to a google function
+            new com.google.common.base.Function<Key<S, T>, T>()
+            {
+                public T apply(final Key<S, T> from)
+                {
+                    return from.get();
+                }
+            });
+
+        ValueMap()
+        {}
+
+        public T get(final Key<S, T> key)
+        {
+            try
+            {
+                return map.get(key);
+            }
+            finally
+            {
+                key.clearReference();
+            }
+        }
+    }
+
+    /**
+     * {@link Key} that can supply a value as well. Needs to have its contained supplier reference cleared after use.
+     */
+    static final class Key<S, T> implements Supplier<T>
+    {
+        private final S two;
+        private volatile Supplier<T> valueSupplier;
+
+        public Key(final S two, final Supplier<T> valueSupplier)
+        {
+            this.two = notNull("two", two);
+            this.valueSupplier = notNull("valueSupplier", valueSupplier);
+        }
+
+        public T get()
+        {
+            if (valueSupplier == null)
+            {
+                throw new IllegalStateException("reference has been cleared already");
+            }
+            return valueSupplier.get();
+        }
+
+        public S getTwo()
+        {
+            return two;
+        }
+
+        void clearReference()
+        {
+            this.valueSupplier = null;
+        }
+
+        @Override
+        public int hashCode()
+        {
+            final int prime = 31;
+            final int result = 1;
+            return prime * result + ((two == null) ? 0 : two.hashCode());
+        }
+
+        @Override
+        public boolean equals(final Object obj)
+        {
+            if (this == obj)
+            {
+                return true;
+            }
+            if (obj == null)
+            {
+                return false;
+            }
+            if (getClass() != obj.getClass())
+            {
+                return false;
+            }
+            return two.equals(((Key<?, ?>) obj).two);
+        }
+    }
+}

File src/main/java/com/atlassian/lucene/stats/cache/DefaultReaderCache.java

+package com.atlassian.lucene.stats.cache;
+
+import com.atlassian.lucene.stats.util.Supplier;
+import org.apache.lucene.index.IndexReader;
+
+import java.util.Collection;
+
+public class DefaultReaderCache implements ReaderCache
+{
+    private final CompositeKeyCache<IndexReader, String, Collection<String>[]> cache = CompositeKeyCache.createWeakFirstKeySoftValueCache();
+
+    public Collection<String>[] get(final IndexReader reader, final String key, final Supplier<Collection<String>[]> supplier)
+    {
+        return cache.get(reader, key, supplier);
+    }
+}

File src/main/java/com/atlassian/lucene/stats/cache/ReaderCache.java

+package com.atlassian.lucene.stats.cache;
+
+import com.atlassian.lucene.stats.util.Supplier;
+import org.apache.lucene.index.IndexReader;
+
+import java.util.Collection;
+
+/**
+ * Cache values by reader.
+ */
+public interface ReaderCache
+{
+    Collection<String>[] get(IndexReader reader, String key, Supplier<Collection<String>[]> supplier);
+}

File src/main/java/com/atlassian/lucene/stats/util/Assertions.java

+package com.atlassian.lucene.stats.util;
+
+/**
+ * Utility class with design by contract checks.
+ */
+public final class Assertions
+{
+    /**
+     * Throw an IllegalArgumentException if the string is null
+     *
+     * @param name added to the exception
+     * @param notNull should not be null
+     * @return argument being checked.
+     * @throws IllegalArgumentException if null
+     */
+    public static <T> T notNull(final String name, final T notNull) throws IllegalArgumentException
+    {
+        if (notNull == null)
+        {
+            throw new NullArgumentException(name);
+        }
+        return notNull;
+    }
+
+
+    static class NullArgumentException extends IllegalArgumentException
+    {
+        NullArgumentException(final String name)
+        {
+            super(name + " should not be null!");
+        }
+    }
+}

File src/main/java/com/atlassian/lucene/stats/util/EasyList.java

+package com.atlassian.lucene.stats.util;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A replacement for UtilMisc.toList().
+ * <p/>
+ * Most methods here are not null safe
+ */
+public class EasyList
+{
+    public static <T> List<T> build(final T o1)
+    {