yt.hub / ythub / uploader.py

from tempfile import TemporaryFile
from flask import jsonify
#from ythub.models import model_registry
from ythub.database import db_session, sdb, bucket
from yt.mods import *
import uuid
import json
from ythub.local_config import tmp_dir
from boto.s3.key import Key
from ythub.utilities import int_to_base36
import random

class UploadHandler(object):
    def __init__(self, chunk_data, metadata):
        self.metadata = metadata
        self.chunk_data = chunk_data
        # Metadata has to tell me a bit about the chunks
        # We now figure out which chunks for each
        # chunk_data is set up like so:
        #   { final_name : whatever,
        #     chunks : [(name, size), (name, size)]
        #   }
        # We assume the chunks will be fed in order
        self.reader = self.chunk_reader()

    def chunk_reader(self):
        chunk_info = {}
        if len(self.chunk_data) is None: raise StopIteration
        if "unique_identifier" in self.metadata:
            uu = self.metadata["unique_identifier"]
        else: # Make our own
            uu = uuid.uuid4().hex
        self.metadata['hub_id'] = uu
        item_name = "%s_%s" % (self.metadata['obj_type'], uu)
        chunk_names = []
        for chunk_name, size in self.chunk_data['chunks']:
            remaining = size
            f = TemporaryFile(dir=tmp_dir)
            yield f
            f.seek(0)
            key = Key(bucket, name = "%s/%s" % (item_name, chunk_name))
            key.set_contents_from_file(f)
            chunk_names.append(chunk_name)
        # When we are all done ...
        self.metadata['chunk_names'] = ";".join(chunk_names)
        self.metadata['created'] = time.time()
        self.metadata['shorturl'] = int_to_base36(int(
            random.random()*2176782335))
        for i, v in sorted(self.metadata.items()):
            print "% 20s = % 60s" % (i, v)
            if type(v) not in [int, float, str, unicode]:
                print "Converting %s (%s) to JSON" % (i, type(v))
                self.metadata[i] = json.dumps(v)
        sdb.put_attributes(item_name, self.metadata)

    def __call__(self, request):
        try:
            f = self.reader.next()
            f.write(request.files['chunk_data'].read())
            return jsonify({'status' : "SENDMORE"})
        except StopIteration:
            url = "https://data.yt-project.org/go/%s" % (
                self.metadata['shorturl'])
            return jsonify({'status' : "COMPLETE",
                            'url' : url})
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.