Anonymous avatar Anonymous committed 3a7a4d5

* Change the signature of constructors/helpers/methods that return an input stream, using '**args'.
Field._getIStreamTags removed and Field.setSubIStream added.
* bz2 & gz parsers: associate a filename to the decompressed stream.
* fat parser: set the size of fragmented streams.
* core: Forgot a InputStream.address in CompressedField (cf [1709])

Comments (0)

Files changed (12)

hachoir-core/doc/hachoir-api.rst

 * createValue(): create value of 'value' attribute
 * createDisplay(): create value of 'display' attribute
 * _createInputStream(): create an InputStream containing the field content
-* _getIStreamTags(): return tags for an InputStream containing the field content
 
 Aliases (method):
 
   if const is True the field set will not be changed
 * __contains__(key)
 * getSubIStream(): return a tagged InputStream containing the field content
+* setSubIStream(): helper to replace _createInputStream (the old one is passed
+  to the new one to allow chaining)
 
 
 Field set class

hachoir-core/hachoir_core/field/field.py

         except FieldError:
             return False
 
-    def _createInputStream(self):
+    def _createInputStream(self, **args):
         assert self._parent
-        return InputFieldStream(self)
-    def _getIStreamTags(self):
-        return []
+        return InputFieldStream(self, **args)
     def getSubIStream(self):
         stream = None
         if hasattr(self, "_sub_istream"):
             stream = self._sub_istream()
         if stream is None:
             stream = self._createInputStream()
-            stream.tags[:0] = self._getIStreamTags()
             self._sub_istream = weakref_ref(stream)
         return stream
+    def setSubIStream(self, createInputStream):
+        cis = self._createInputStream
+        self._createInputStream = lambda **args: createInputStream(cis, **args)
 
     def __nonzero__(self):
         """

hachoir-core/hachoir_core/field/link.py

             error(str(e))
         return None
 
-    def _createInputStream(self):
+    def _createInputStream(self, **args):
         if self.first is self and hasattr(self.first, "_getData"):
-            return FragmentedStream(self.first)
-        return FieldSet._createInputStream(self)
+            return FragmentedStream(self.first, **args)
+        return FieldSet._createInputStream(self, **args)
 
     def _createNext(self):
         return None

hachoir-core/hachoir_core/field/sub_file.py

             if not description:
                 description = 'File "%s" (%s)' % (filename, humanFilesize(length))
         Bytes.__init__(self, parent, name, length, description)
-        tags = []
-        if parser is not None:
-            tags.append(( "id", parser.tags["id"]))
-        if mime_type:
-            tags.append(( "mime", mime_type ))
-        if filename:
-            tags.append(( "filename", filename ))
-        self._getIStreamTags = lambda: tags
+        def createInputStream(cis, **args):
+            tags = args.setdefault("tags",[])
+            if parser is not None:
+                tags.append(( "id", parser.tags["id"]))
+            if mime_type:
+                tags.append(( "mime", mime_type ))
+            if filename:
+                tags.append(( "filename", filename ))
+            return cis(**args)
+        self.setSubIStream(createInputStream)
 
 class CompressedStream:
     offset = 0
         self._buffer = d[size+len(d):]
         return ''.join(data)
 
-def CompressedField(field, decompressor, size=None):
-    if field._parent:
-        cis = field._createInputStream
-    else:
-        cis = lambda: field.stream
-    def createInputStream(size=size):
-        stream = cis()
+def CompressedField(field, decompressor):
+    def createInputStream(cis, source=None, **args):
+        if field._parent:
+            stream = cis(source=source)
+            args.setdefault("tags", []).extend(stream.tags)
+        else:
+            stream = field.stream
         input = CompressedStream(stream, decompressor)
-        address = field.absolute_address
-        source = "Compressed source: '%s' (offset=%s)" % (stream.source, address)
-        if callable(size):
-            size = size()
-        stream = InputIOStream(input, source, size)
-        stream.address = address
-        return stream
-    field._createInputStream = createInputStream
+        if source is None:
+            source = "Compressed source: '%s' (offset=%s)" % (stream.source, field.absolute_address)
+        return InputIOStream(input, source=source, **args)
+    field.setSubIStream(createInputStream)
     return field

hachoir-core/hachoir_core/stream/input.py

     _set_size = None
     _current_size = 0
 
-    def __init__(self, source=None, size=None):
+    def __init__(self, source=None, size=None, **args):
         self.source = source
         self._size = size   # in bits
         if size == 0:
             raise NullStreamError(source)
-        self.tags = [ ]
+        self.tags = args.get("tags", [])
 
     def askSize(self, client):
         if self._size != self._current_size:
         return data
 
 class InputIOStream(InputStream):
-    def __init__(self, input, source=None, size=None):
+    def __init__(self, input, size=None, **args):
         if not hasattr(input, "seek"):
             if size is None:
                 input = InputPipe(input, self._setSize)
                     errmsg = unicode(str(err), charset)
                     raise InputStreamError(_("Unable to get size of %s: %s") % (source, errmsg))
         self._input = input
-        InputStream.__init__(self, source, size)
+        InputStream.__init__(self, size=size, **args)
 
     def __current_size(self):
         if self._size:
 
 
 class InputSubStream(InputStream):
-    def __init__(self, stream, offset, size=None, source=None):
+    def __init__(self, stream, offset, size=None, source=None, **args):
         if size is None and stream.size is not None:
             size = stream.size - offset
         if None < size <= 0:
         self.stream = stream
         self._offset = offset
         if source is None:
-            source = "<substream input=%s offset=%s size=%s>" % \
-                (stream.source, offset, size)
-        InputStream.__init__(self, source, size)
+            source = "<substream input=%s offset=%s size=%s>" % (stream.source, offset, size)
+        InputStream.__init__(self, source=source, size=size, **args)
         self.stream.askSize(self)
 
     _current_size = property(lambda self: min(self._size, max(0, self.stream._current_size - self._offset)))
     def read(self, address, size):
         return self.stream.read(self._offset + address, size)
 
-def InputFieldStream(field):
+def InputFieldStream(field, **args):
     if not field.parent:
         return field.stream
     stream = field.parent.stream
-    return InputSubStream(stream, field.absolute_address, field.size, stream.source + field.path)
+    args["size"] = field.size
+    args.setdefault("source", stream.source + field.path)
+    return InputSubStream(stream, field.absolute_address, **args)
 
 
 class FragmentedStream(InputStream):
-    def __init__(self, field, size=None):
+    def __init__(self, field, **args):
         self.stream = field.parent.stream
         data = field.getData()
         self.fragments = [ (0, data.absolute_address, data.size) ]
         self.next = field.next
-        InputStream.__init__(self, self.stream.source + field.path, size)
+        args.setdefault("source", "%s%s" % (self.stream.source, field.path))
+        InputStream.__init__(self, **args)
         if not self.next:
             self._current_size = data.size
             self._setSize()
 
 class ConcatStream(InputStream):
     # TODO: concatene any number of any type of stream
-    def __init__(self, streams, source=None):
+    def __init__(self, streams, **args):
         if len(streams) > 2 or not streams[0].checked:
             raise NotImplementedError
         self.__size0 = streams[0].size
-        size = streams[1].askSize(self)
-        if size is not None:
-            size += self.__size0
+        size1 = streams[1].askSize(self)
+        if size1 is not None:
+            args["size"] = self.__size0 + size1
         self.__streams = streams
-        InputStream.__init__(self, source, size)
+        InputStream.__init__(self, **args)
 
     _current_size = property(lambda self: self.__size0 + self.__streams[1]._current_size)
 

hachoir-core/hachoir_core/stream/input_helper.py

 from hachoir_core.stream import InputIOStream, InputStreamError
 from cStringIO import StringIO
 
-def StringInputStream(content, source="<string>"):
+def StringInputStream(content, source="<string>", **args):
     inputio = StringIO(content)
-    return InputIOStream(inputio, source, size=len(content)*8)
+    args["size"] = len(content) * 8
+    return InputIOStream(inputio, source=source, **args)
 
 
 def FileInputStream(filename, real_filename=None):
         errmsg = unicode(str(err), charset)
         raise InputStreamError(_("Unable to open file %s: %s")
             % (filename, errmsg))
-    stream = InputIOStream(inputio, "file:%s" % filename)
-    stream.tags.append(("filename", filename))
-    return stream
+    return InputIOStream(inputio, source="file:" + filename, tags=[ ("filename", filename) ])

hachoir-parser/hachoir_parser/archive/bzip2_parser.py

 
 from hachoir_parser import Parser
 from hachoir_core.field import (ParserError, String,
-    Character, UInt8, UInt32, SubFile, CompressedField)
+    Bytes, Character, UInt8, UInt32, CompressedField)
 from hachoir_core.endian import LITTLE_ENDIAN
 from hachoir_core.text_handler import hexadecimal
 
 
         size = (self._size - self.current_size)/8
         if size:
-            try:
-                filename = self.stream.filename
-                if filename and filename.endswith(".bz2"):
+            for tag, filename in self.stream.tags:
+                if tag == "filename" and filename.endswith(".bz2"):
                     filename = filename[:-4]
-                else:
-                    filename = None
-            except AttributeError:
+                    break
+            else:
                 filename = None
-            data = SubFile(self, "file", size, filename=filename)
+            data = Bytes(self, "file", size)
             if has_deflate:
                 CompressedField(self, Bunzip2)
-                data._createInputStream = lambda: self._createInputStream()
+                def createInputStream(**args):
+                    if filename:
+                        args.setdefault("tags",[]).append(("filename", filename))
+                    return self._createInputStream(**args)
+                data._createInputStream = createInputStream
             yield data
 

hachoir-parser/hachoir_parser/archive/gzip_parser.py

             if self["has_filename"].value:
                 filename = self["filename"].value
             else:
-                filename = None
+                for tag, filename in self.stream.tags:
+                    if tag == "filename" and filename.endswith(".gz"):
+                        filename = filename[:-3]
+                        break
+                else:
+                    filename = None
             data = SubFile(self, "file", size, filename=filename)
             if has_deflate:
-                data = CompressedField(data, Gunzip)
+                CompressedField(data, Gunzip)
             yield data
 
         # Footer

hachoir-parser/hachoir_parser/container/mkv.py

         if filename:
             return 'File "%s"' % filename
         return "('Filename' entry not found)"
-    def _getIStreamTags(self):
+    def _createInputStream(self, **args):
+        tags = args.setdefault("tags",[])
         try:
-            mime = self["../../FileMimeType/string"]
+            tags.append(("mime", self["../../FileMimeType/string"].value))
         except MissingField:
-            tags = [ ]
-        else:
-            tags = [ ("mime", mime.value) ]
+            pass
         filename = self._getFilename()
         if filename:
             tags.append(("filename", filename))
-        return tags
+        return Bytes._createInputStream(self, **args)
 
 def UTF8(parent):
     return _String(parent,'unicode', parent['size'].value, charset='UTF-8')

hachoir-parser/hachoir_parser/container/swf.py

 from hachoir_parser import Parser
 from hachoir_core.field import (FieldSet, ParserError,
     Bit, Bits, UInt8, UInt32, UInt16, CString, Enum,
-    Bytes, RawBytes, NullBits, String, SubFile, CompressedField)
+    Bytes, RawBytes, NullBits, String, CompressedField)
 from hachoir_core.endian import LITTLE_ENDIAN, BIG_ENDIAN
 from hachoir_core.text_handler import humanFilesize
 from hachoir_core.tools import paddingSize, humanFrequency
         else:
             size = (self.size - self.current_size) // 8
             if has_deflate:
-                data = CompressedField(SubFile(self, "compressed_data", size, parser=SwfFile), Gunzip)
-                cis = data._createInputStream
-                def createInputStream():
-                    stream = cis()
+                data = CompressedField(Bytes(self, "compressed_data", size), Gunzip)
+                def createInputStream(cis, source=None, **args):
+                    stream = cis(source=source)
                     header = StringInputStream("FWS" + self.stream.readBytes(3*8, 5))
-                    return ConcatStream((header, stream), stream.source)
-                data._createInputStream = createInputStream
+                    args.setdefault("tags",[]).append(("id", self.tags["id"]))
+                    return ConcatStream((header, stream), source=stream.source, **args)
+                data.setSubIStream(createInputStream)
                 yield data
             else:
                 yield Bytes(self, "compressed_data", size)

hachoir-parser/hachoir_parser/file_system/fat.py

         self.path = path
         self.filesize = entry.target_size
         self.done = 0
-        self.tags = lambda: [ ("filename", entry.getFilename()) ]
+        def createInputStream(cis, **args):
+            args["size"] = self.filesize
+            args.setdefault("tags",[]).append(("filename", entry.getFilename()))
+            return cis(**args)
+        self.createInputStream = createInputStream
 
     def __call__(self, prev):
         name = self.path + "[]"
                 return
             field = File(self.root, name, size=size, first=prev.first)
             if prev.first is None:
-                field._getIStreamTags = self.tags
                 field._description = 'File size: %s' % humanFilesize(self.filesize//8)
+                field.setSubIStream(self.createInputStream)
             field.datasize = min(self.filesize - self.done, size)
             self.done += field.datasize
         else:

hachoir-parser/hachoir_parser/guess.py

     if offset or size:
         if size:
             size *= 8
-        stream = InputSubStream(stream, 8 * max(0, offset), size)
+        stream = InputSubStream(stream, 8 * max(0, offset), size=size)
 
     # Create field set
     if real_filename:
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.