Commits

Thomas Figg committed 8199137 Merge

merge

Comments (0)

Files changed (2)

hanzo/warctools/__init__.py

 from .arc import ArcRecord
 from .mixed import MixedRecord
 from .s3 import list_files
-from . import record, warc, arc
+from . import record, warc, arc, s3
 
 def expand_files(files):
     for file in files:

hanzo/warctools/s3.py

 from urlparse import urlparse
 
-
-import tempfile
 from cStringIO import StringIO
 
-
 try:
-    from boto.s3.connection import S3Connection, Location
+    from boto.s3.connection import S3Connection
     from boto.s3.key import Key
 except ImportError:
     def open_url(url, offset=None, length=None):
         bucket_name = p.netloc
         key = p.path[1:]
         conn = S3Connection()
-        b= conn.get_bucket(bucket_name)
-        k = Key(b)
+        bucket = conn.get_bucket(bucket_name)
+        k = Key(bucket)
         k.key = key
+        if offset is not None and length is not None:
+            headers = {'Range': 'bytes=%d-%d' % (offset, offset + length)}
+        elif offset is not None:
+            headers = {'Range': 'bytes=%d-' % offset}
+        else:
+            headers = {}
+
         s = StringIO()
-        if offset is not None and length is not None:
-            headers={'Range' : 'bytes=%d-%d'%(offset, offset+length)}
-        elif offset is not None:
-            headers={'Range' : 'bytes=%d-'%(offset)}
-        else:
-            headers={}
         k.get_contents_to_file(s, headers=headers)
         s.seek(0)
         return s
 
         conn = S3Connection()
 
-        b= conn.get_bucket(bucket_name)
+        bucket = conn.get_bucket(bucket_name)
         complete  = False
         marker = ''
 
         while not complete:
-            rs = b.get_all_keys(prefix=prefix, marker=marker, delimiter='')
+            rs = bucket.get_all_keys(prefix=prefix, marker=marker, delimiter='')
             for k in rs:
-                yield 's3://%s/%s'%(bucket_name, k.key)
+                yield 's3://%s/%s' % (bucket_name, k.key)
                 marker = k.key
 
             complete = not rs.is_truncated
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.