DateRange query fails if field contains None

Issue #77 resolved
Nikolaus Rath
created an issue

If a document has been stored with expiration_date=None, the query DateRange('expiration_date', datetime.fromtimestamp(0), datetime.today())]) fails with

{{{ Traceback (most recent call last): File "/home/nikratio/projekte/archiver/archive_expire.py", line 112, in <module> main(sys.argv[1:]) File "/home/nikratio/projekte/archiver/archive_expire.py", line 71, in main for document in searcher.search(query): File "/home/nikratio/.local/lib/python2.6/site-packages/Whoosh-1.3.3-py2.6.egg/whoosh/searching.py", line 378, in search matcher = query.matcher(self) File "/home/nikratio/.local/lib/python2.6/site-packages/Whoosh-1.3.3-py2.6.egg/whoosh/query.py", line 556, in matcher return self.matcher(IntersectionMatcher, searcher, exclude_docs) File "/home/nikratio/.local/lib/python2.6/site-packages/Whoosh-1.3.3-py2.6.egg/whoosh/query.py", line 408, in _matcher submatchers = self._submatchers(searcher, exclude_docs) File "/home/nikratio/.local/lib/python2.6/site-packages/Whoosh-1.3.3-py2.6.egg/whoosh/query.py", line 402, in _submatchers subs.sort(key=lambda q: q.estimate_size(r)) File "/home/nikratio/.local/lib/python2.6/site-packages/Whoosh-1.3.3-py2.6.egg/whoosh/query.py", line 402, in <lambda> subs.sort(key=lambda q: q.estimate_size(r)) File "/home/nikratio/.local/lib/python2.6/site-packages/Whoosh-1.3.3-py2.6.egg/whoosh/query.py", line 1070, in estimate_size return self._compile_query(ixreader).estimate_size(ixreader) File "/home/nikratio/.local/lib/python2.6/site-packages/Whoosh-1.3.3-py2.6.egg/whoosh/query.py", line 273, in estimate_size return self.child.estimate_size(ixreader) File "/home/nikratio/.local/lib/python2.6/site-packages/Whoosh-1.3.3-py2.6.egg/whoosh/query.py", line 594, in estimate_size return sum(q.estimate_size(ixreader) for q in self.subqueries) File "/home/nikratio/.local/lib/python2.6/site-packages/Whoosh-1.3.3-py2.6.egg/whoosh/query.py", line 594, in <genexpr> return sum(q.estimate_size(ixreader) for q in self.subqueries) File "/home/nikratio/.local/lib/python2.6/site-packages/Whoosh-1.3.3-py2.6.egg/whoosh/query.py", line 454, in estimate_size for text in self._words(ixreader)) File "/home/nikratio/.local/lib/python2.6/site-packages/Whoosh-1.3.3-py2.6.egg/whoosh/query.py", line 453, in <genexpr> return sum(ixreader.doc_frequency(self.fieldname, text) File "/home/nikratio/.local/lib/python2.6/site-packages/Whoosh-1.3.3-py2.6.egg/whoosh/query.py", line 987, in _words for fname, t, , _ in ixreader.iter_from(fieldname, self.start): File "/home/nikratio/.local/lib/python2.6/site-packages/Whoosh-1.3.3-py2.6.egg/whoosh/filedb/filereading.py", line 174, in iter_from for (fn, t), (totalfreq, _, postcount) in self.termsindex.items_from((fieldname, text)): File "/home/nikratio/.local/lib/python2.6/site-packages/Whoosh-1.3.3-py2.6.egg/whoosh/filedb/filetables.py", line 444, in items_from fromkey = self.keycoder(key) File "/home/nikratio/.local/lib/python2.6/site-packages/Whoosh-1.3.3-py2.6.egg/whoosh/filedb/filetables.py", line 525, in keycoder return pack_ushort(self.fieldmap[key[0]]) + utf8encode(key[1])[0] KeyError: u'expiration_date'

}}}

Comments (3)

  1. Nikolaus Rath reporter

    Here's a short testcase:

    #!/usr/bin/env python
    
    from __future__ import unicode_literals, division, print_function, absolute_import
    
    import tempfile
    import atexit
    import shutil
    import whoosh.index
    import whoosh.fields
    from datetime import datetime
    from whoosh.fields import Schema
    from whoosh.query import DateRange
    
    tempdir = tempfile.mkdtemp()
    atexit.register(lambda : shutil.rmtree(tempdir))
    
    
    schema = Schema(test=whoosh.fields.DATETIME)
                            
    ix = whoosh.index.create_in(tempdir, schema)
    writer = ix.writer()
    writer.add_document(test = None)
    writer.commit()
    
    print(whoosh.versionstring())
    searcher = ix.searcher()
    query = DateRange('test', 
                      datetime.fromtimestamp(0),
                      datetime.today())
    searcher.search(query)
    

    This gives:

    1.3.3
    Traceback (most recent call last):
      File "/home/nikratio/projekte/archiver/test.py", line 30, in <module>
        searcher.search(query)
      File "/home/nikratio/.local/lib/python2.6/site-packages/Whoosh-1.3.3-py2.6.egg/whoosh/searching.py", line 378, in search
        matcher = query.matcher(self)
      File "/home/nikratio/.local/lib/python2.6/site-packages/Whoosh-1.3.3-py2.6.egg/whoosh/query.py", line 1111, in matcher
        return q.matcher(searcher, exclude_docs=exclude_docs)
      File "/home/nikratio/.local/lib/python2.6/site-packages/Whoosh-1.3.3-py2.6.egg/whoosh/query.py", line 1372, in matcher
        m = self.child.matcher(searcher, exclude_docs=None)
      File "/home/nikratio/.local/lib/python2.6/site-packages/Whoosh-1.3.3-py2.6.egg/whoosh/query.py", line 603, in matcher
        return self._matcher(UnionMatcher, searcher, exclude_docs)
      File "/home/nikratio/.local/lib/python2.6/site-packages/Whoosh-1.3.3-py2.6.egg/whoosh/query.py", line 408, in _matcher
        submatchers = self._submatchers(searcher, exclude_docs)
      File "/home/nikratio/.local/lib/python2.6/site-packages/Whoosh-1.3.3-py2.6.egg/whoosh/query.py", line 402, in _submatchers
        subs.sort(key=lambda q: q.estimate_size(r))
      File "/home/nikratio/.local/lib/python2.6/site-packages/Whoosh-1.3.3-py2.6.egg/whoosh/query.py", line 402, in <lambda>
        subs.sort(key=lambda q: q.estimate_size(r))
      File "/home/nikratio/.local/lib/python2.6/site-packages/Whoosh-1.3.3-py2.6.egg/whoosh/query.py", line 454, in estimate_size
        for text in self._words(ixreader))
      File "/home/nikratio/.local/lib/python2.6/site-packages/Whoosh-1.3.3-py2.6.egg/whoosh/query.py", line 453, in <genexpr>
        return sum(ixreader.doc_frequency(self.fieldname, text)
      File "/home/nikratio/.local/lib/python2.6/site-packages/Whoosh-1.3.3-py2.6.egg/whoosh/query.py", line 987, in _words
        for fname, t, _, _ in ixreader.iter_from(fieldname, self.start):
      File "/home/nikratio/.local/lib/python2.6/site-packages/Whoosh-1.3.3-py2.6.egg/whoosh/filedb/filereading.py", line 174, in iter_from
        for (fn, t), (totalfreq, _, postcount) in self.termsindex.items_from((fieldname, text)):
      File "/home/nikratio/.local/lib/python2.6/site-packages/Whoosh-1.3.3-py2.6.egg/whoosh/filedb/filetables.py", line 444, in items_from
        fromkey = self.keycoder(key)
      File "/home/nikratio/.local/lib/python2.6/site-packages/Whoosh-1.3.3-py2.6.egg/whoosh/filedb/filetables.py", line 525, in keycoder
        return pack_ushort(self.fieldmap[key[0]]) + utf8encode(key[1])[0]
    KeyError: u'test'
    
  2. Log in to comment