Commits

Miki Tebeka committed 0c48b63 Merge

Merge with default

  • Participants
  • Parent commits 0b51cdd, 22999de
  • Branches write

Comments (0)

Files changed (13)

 2bb6c8d39c9c9b3c2a56b1373044b6725bf6a244 0.6.8
 bcad6af1a1ed05147aca78b8e2171f3e34c6f1f7 0.6.9
 75c9094ed3c6afd4c648ae2a71c3b91fe3cbd93b 0.6.10
+69c85b797bc2fa09413e1907332114ae3948e8a3 0.7.0
+69c85b797bc2fa09413e1907332114ae3948e8a3 0.7.0
+e2703fb77fb6e79413d27e7fb2b02477eed161ae 0.7.0
+ca548b5a2557e8e9274015a8889e0c0cd8bf7932 0.7.1
+2012-12-10 version 0.7.1
+    * Fixed README (about snappy)
+
+2012-12-10 version 0.7.0
+    * Snappy support
+
 2012-10-05 version 0.6.10
     * Handle names in unions (aboSamoor)
     * Handle namespaces (aboSamoor)
 ===========
 * Support only iteration
     - No writing for you!
-* Supports only `null` and `deflate` codecs
-    - `avro` also supports `snappy`
 * No reader schema
 
 Hacking

File fastavro/__init__.py

 '''
 
 __all__ = [ 'iter_avro', 'reader']
-__version__ = '0.6.10'
+__version__ = '0.7.1'
 
 
 try:

File fastavro/__main__.py

 
     parser = ArgumentParser(
         description='iter over avro file, emit records as JSON')
-    parser.add_argument('file', help='file(s) to parse', nargs='+')
+    parser.add_argument('file', help='file(s) to parse', nargs='*')
     parser.add_argument('--schema', help='dump schema instead of records',
                         action='store_true', default=False)
+    parser.add_argument('--codecs', help='print supported codecs',
+                        action='store_true', default='false')
     parser.add_argument('--version', action='version',
             version='fastavro {0}'.format(avro.__version__))
     args = parser.parse_args(argv[1:])
 
-    for filename in args.file:
+    if args.codecs:
+        import fastavro
+        print('\n'.join(sorted(fastavro._reader.BLOCK_READERS)))
+        raise SystemExit
+
+    files = args.file or ['-']
+    for filename in files:
         if filename == '-':
             fo = sys.stdin
         else:

File fastavro/reader.py

     if mark != sync_marker:
         fo.seek(-SYNC_SIZE, SEEK_CUR)
 
-
 def null_read_block(fo):
     '''Read block in "null" codec.'''
     read_long(fo, None)
     return fo
 
-
 def deflate_read_block(fo):
     '''Read block in "deflate" codec.'''
     data = read_bytes(fo, None)
     'deflate': deflate_read_block
 }
 
+try:
+    import snappy
+    def snappy_read_block(fo):
+        length = read_long(fo, None)
+        data = fo.read(length - 4)
+        fo.read(4) # CRC
+        return MemoryIO(snappy.decompress(data))
+
+    BLOCK_READERS['snappy'] = snappy_read_block
+except ImportError:
+    pass
 
 def _iter_avro(fo, header, schema):
     '''Return iterator over avro records.'''
 make
 python setup.py bdist_egg upload
 python3 setup.py bdist_egg upload
+python3.3 setup.py bdist_egg upload
 python setup.py sdist upload
 rm -fr build dist
 hg tag -f $(python setup.py --version)
         'Topic :: Software Development :: Libraries',
         'Topic :: Software Development :: Libraries :: Python Modules',
     ],
-    install_requires=install_requires
+    install_requires=install_requires,
+    extras_require = {
+        'snappy': ['python-snappy'],
+    },
 )

File test-install.sh

 $python setup.py install
 (cd /tmp && python -c 'import fastavro; print(fastavro.iter_avro)')
 (cd /tmp && fastavro --help)
+(cd /tmp && fastavro --codecs)

File tests/avro-files/snappy.avro

Binary file added.

File tests/avro-files/test-snappy.avro

Binary file added.

File tests/avro-files/weather-snappy.avro

Binary file added.

File tests/test_fastavro.py

 
 data_dir = join(abspath(dirname(__file__)), 'avro-files')
 
+try:
+    import snappy
+    has_snappy = True
+except ImportError:
+    has_snappy = False
+
 NO_DATA = set([
     'class org.apache.avro.tool.TestDataFileTools.zerojsonvalues.avro',
     'testDataFileMeta.avro',
 
 def test_fastavro():
     for filename in iglob(join(data_dir, '*.avro')):
+        if (not has_snappy) and ('snappy' in filename):
+            continue
         yield check, filename