1. scorpil
  2. warc-tools

Commits

Thomas Figg  committed 1573678

trailing newlines check

  • Participants
  • Parent commits dd8877f
  • Branches default

Comments (0)

Files changed (2)

File arc2warc.py

View file
  • Ignore whitespace
         try:
             for record in fh:
                 if isinstance(record, WarcRecord):
-                    print >> sys.stderr, 'warcrecord found'
+                    print >> sys.stderr, '   WARC', record.url
                     warcs = [record]
                 else:
+                    print >> sys.stderr, 'ARC    ', record.url
                     warcs = arc.convert(record)
 
                 for warcrecord in warcs:

File hanzo/warctools/mixed.py

View file
  • Ignore whitespace
 
     def parse(self, stream, offset=None):
         line = stream.readline()
-        if line.startswith('WARC'):
-            record, errors, offset = self.warc.parse(stream, offset, line=line)
-            errors = list(errors) + list(self.warc.trim(stream))
-        else:
-            record, errors, offset = self.arc.parse(stream, offset, line=line)
+        while line:
+            if line.startswith('WARC'):
+                return self.warc.parse(stream, offset, line=line)
+            elif line not in ('\n','\r\n','\r'):
+                return self.arc.parse(stream, offset, line=line)
 
-        return record, errors,offset
+            line = stream.readline()
+        return None, (), offset
 
+