Anonymous avatar Anonymous committed 623885b

Implemented FLVConverter

Comments (0)

Files changed (8)

 # WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*
 # Debug mode will enable the interactive debugging tool, allowing ANYONE to
 # execute malicious code after an exception is raised.
-set debug = false
+#set debug = false
 
 upload_dir = %(here)s/data/uploads
 

skypieamc/config/environment.py

 from skypieamc.config.routing import make_map
 from skypieamc.model import init_model
 from skypieamc.model.filemanager import FileManager
+from skypieamc.lib.flvconverter import FLVConverter
 
 def load_environment(global_conf, app_conf):
     """Configure the Pylons environment via the ``pylons.config``
     # any Pylons config options)
     config['pylons.strict_c'] = True
     config['pylons.app_globals'].upload_dir = app_conf['upload_dir']
+
     config['pylons.app_globals'].file_manager = FileManager()
     config['pylons.app_globals'].file_manager.daemon = True
     config['pylons.app_globals'].file_manager.start()
+
+    config['pylons.app_globals'].flv_converter = FLVConverter()
+    config['pylons.app_globals'].flv_converter.daemon = True
+    config['pylons.app_globals'].flv_converter.start()

skypieamc/controllers/file.py

             safename = h.safe_filename(ifile.filename)
             pfile_path = os.path.join(dir, safename)
             if not os.path.exists(pfile_path):
+                # Copy file to upload directory
+                with open(pfile_path, 'wb') as pfile:
+                    try:
+                        shutil.copyfileobj(ifile.file, pfile)
+                    except (IOError, os.error) as why:
+                        abort(503, u'Unable to copy file')
                 break
-
-        # Copy file to upload directory
-        with open(pfile_path, 'wb') as pfile:
-            try:
-                shutil.copyfileobj(ifile.file, pfile)
-            except (IOError, os.error) as why:
-                abort(503, u'Unable to copy file')
         ifile.file.close()
 
         # Add File instance to db
         mfile.size = os.stat(pfile_path).st_size
         meta.Session.add(mfile)
         meta.Session.commit()
+        
+        # Convert to .flv
+        app_globals.flv_converter.add_job(mfile)
 
         session['flash_msg'] = u'File successfully uploaded!'
         session.save()
 
         id -- int/str/unicode
         """
-        mfile = app_globals.file_manager.download_request(id)
+        mfile = id and meta.Session.query(model.File).get(id)
         if mfile is None:
             abort(404, u'File not found')
+        pfile_name = mfile.safename
+        askfor_playable = request.params.has_key('playable')
+        if askfor_playable:
+            pfile_name = mfile.playable
         try:
             pfile = open(
-                os.path.join(app_globals.upload_dir, mfile.safename),
+                os.path.join(app_globals.upload_dir, pfile_name),
                 'rb'
             )
         except IOError as why:
             abort(404, u'File cannot be read')
 
-        response.content_type = guess_type(mfile.name)[0] or 'text/plain'
-        response.content_disposition = 'attachment; filename={0}'.\
-            format(mfile.name.encode('utf-8'))
+        if not askfor_playable:
+            response.content_type = guess_type(mfile.name)[0] or 'text/plain'
+            if not h.file_type(mfile.name) == 'image':
+                response.content_disposition = 'attachment; filename="{0}"'.\
+                    format(mfile.name.encode('utf-8'))
 
         def stream_content():
             """
                     yield data
                 else:
                     pfile.close()
+                    app_globals.file_manager.download_count_up(mfile.id)
                     break
 
-        if config['debug']:
+        if config['debug'] and not askfor_playable:
             data = pfile.read()
             pfile.close()
+            app_globals.file_manager.download_count_up(mfile.id)
             return data
         else:
             return stream_content()
         id -- int/str/unicode
         """
         c.play_url = h.url_for(controller='file', action='download', id=id)
+        c.play_url += '?playable'
         return render('/derived/file/play.mako')

skypieamc/lib/flvconverter.py

+"""FLVConverter"""
+import logging
+import os.path
+import subprocess
+import threading
+import time
+
+from heapq import heappush, heappop
+
+from pylons import config
+
+from skypieamc.model import meta
+from skypieamc.lib import helpers as h
+
+log = logging.getLogger(__name__)
+
+class FLVConverter(threading.Thread):
+
+    __max_workers = 5
+
+    class __Worker(threading.Thread):
+
+        def __init__(self, jobs, lock, job_cond, *args, **kwargs):
+            threading.Thread.__init__(self, *args, **kwargs)
+            self._jobs = jobs
+            self._lock = lock
+            self._job_available = job_cond
+
+        def run(self):
+            while True:
+                mfile = None
+                with self._lock:
+                    self._job_available.wait()
+                    # When I woke up, there should be a job
+                    mfile = heappop(self._jobs)[1]
+                self.process_job(mfile)
+
+        __cmdexts = {
+            'ffmpeg -i {0} -acodec libfaac -ar 44100 -ab 128k -f flv {1}':
+                ['.mp3', '.flac', '.ape'],
+            'ffmpeg -i {0} -acodec libfaac -ar 48000 -ab 96k -sameq -f flv {1}':
+                ['.avi', '.mkv', '.mp4'],
+        }
+
+        def process_job(self, mfile):
+            """
+            Convert the underlying file of mfile to FLV if possible
+            
+            mfile -- File model instance
+            """
+            ext = os.path.splitext(mfile.name)[1].lower()
+            for cmd, exts in self.__cmdexts.iteritems():
+                if ext in exts:
+                    dir = config['app_conf']['upload_dir']
+                    name = u'FLV' + mfile.name
+                    playable = None
+                    pfile_path = None
+                    while True:
+                        playable = h.safe_filename(name)
+                        pfile_path = os.path.join(dir, playable)
+                        if not os.path.exists(pfile_path):
+                            mfile_path = os.path.join(dir, mfile.safename)
+                            log.info('flv_worker{0}: Start processing {1}({2})...'.\
+                                format(self.ident, mfile.id, mfile.name.encode('utf-8')))
+                            retval = subprocess.call(
+                                cmd.format(mfile_path, pfile_path),
+                                shell=True
+                            )
+                            if retval:
+                                os.remove(pfile_path)
+                                log.info('flv_worker{0}: Processing {1}({2}) failed!'.\
+                                    format(self.ident, mfile.id, mfile.name.encode('utf-8')))
+                            else:
+                                mfile.playable = playable
+                                # Need to add mfile to Seesion since this code
+                                # is executing by our worker thread, not the
+                                # pylons thread.
+                                meta.Session.add(mfile)
+                                meta.Session.commit()
+                                log.info('flv_worker{0}: {1}({2}) done!'.\
+                                    format(self.ident, mfile.id, mfile.name.encode('utf-8')))
+                            break
+
+    def __init__(self, *args, **kwargs):
+        """
+        Initiate locks and job queues
+        """
+        threading.Thread.__init__(self, *args, **kwargs)
+        self._jobqueue = [] #[(fsize1, file1), (fsize2, file2)...]
+        self._jobqueue_lock = threading.Lock()
+        self._job_available = threading.Condition(self._jobqueue_lock)
+        self._workers = []
+        for i in xrange(self.__max_workers):
+            worker = self.__Worker(self._jobqueue,
+                                   self._jobqueue_lock,
+                                   self._job_available)
+            worker.start()
+            self._workers.append(worker)
+
+    def run(self):
+        """
+        Perform administrative tasks in every 1 minute
+        """
+        while True:
+            with self._jobqueue_lock:
+                for i in xrange(len(self._jobqueue)):
+                    self._job_available.notify()
+            time.sleep(60)
+
+    def add_job(self, mfile=None):
+        """
+        Add a job to job queue
+        
+        mfile -- File model instance (this instance should be added and
+                 committed to db already)
+        """
+        if mfile is None:
+            return
+        with self._jobqueue_lock:
+            heappush(self._jobqueue, (mfile.size, mfile))

skypieamc/lib/helpers.py

 Consists of functions to typically be used within templates, but also
 available to Controllers. This module is available to templates as 'h'.
 """
+import os.path
 import random
 
 from base64 import urlsafe_b64encode, urlsafe_b64decode
     filename = filename[:16] + str(int(random.random() * 10 ** 8))
     return encode_filename(filename)
 
+__sizeunits = [
+    (1024.0 ** 3, 'GB'),
+    (1024.0 ** 2, 'MB'),
+    (1024.0 ** 1, 'KB'),
+    (1024.0 ** 0, 'B'),
+]
 def readable_filesize(fsize):
     """
     Produce a human readable file size, e.g., 1.0 KB, 128.0 MB, 4.4 GB
     size -- int
     return -- unicode
     """
-    units = [
-        (1024.0 ** 3, 'GB'),
-        (1024.0 ** 2, 'MB'),
-        (1024.0 ** 1, 'KB'),
-        (1024.0 ** 0, 'B'),
-    ]
-
-    for s, u in units:
+    for s, u in __sizeunits:
         result = fsize / s
         if round(result) > 0:
             return u'{0:.1f} {1}'.format(result, u)
 
-def is_playable(filename):
+__typeexts = {
+    'video': ['.avi', '.mp4', '.mkv', '.rmvb', '.flv', '.wmv'],
+    'audio': ['.mp3', '.wav', '.flac', '.ape', '.wma'],
+    'image': ['.bmp', '.gif', '.jpg', '.jpeg', '.png'],
+    'package': ['.bz2', '.gz', '.tar', '.zip', '.rar', '.7z'],
+    'file': None
+}
+def file_type(fname):
     """
-    Determine whether a file is a supported playable type from the file name
+    Determine the type of a file
     
-    filename -- str/unicode
+    fname -- str/unicode
+    return -- 'video'/'audio'/'image'/'package'/'file'
     """
-    return False
+    ext = os.path.splitext(fname)[1].lower()
+    for type in __typeexts:
+        if __typeexts[type] is not None and \
+           ext in __typeexts[type]:
+            return type
+    else:
+        return 'file'

skypieamc/model/__init__.py

     schema.Column('size', types.Integer(), nullable=False),
     schema.Column('uploaded', types.DateTime(), default=h.now),
     schema.Column('download_count', types.Integer(), default=0),
+    schema.Column('playable', types.String(127)),
 )
 tag_table = schema.Table('tag', meta.metadata,
     schema.Column('id', types.Integer(),

skypieamc/model/filemanager.py

                         del self._filelock_map[id]
             time.sleep(60)
 
-    def download_request(self, id=None):
+    def download_count_up(self, id=None):
         """
-        Increase the file's download_count by 1 and return
-        the File model instance
+        Increase the file's download_count by 1
 
         id -- int/str/unicode
-        return File/None
         """
+        if id is None:
+          return
         id = int(id)
         filelock = self._acquire_filelock(id)
         with filelock:
-            mfile = id and meta.Session.query(model.File).get(id)
+            mfile = meta.Session.query(model.File).get(id)
             if mfile is None:
-                return None
+                self._release_filelock(id)
+                return
             mfile.download_count += 1
             meta.Session.commit()
             self._release_filelock(id)
-            return mfile
 
     def _acquire_filelock(self, id):
         """
     def _release_filelock(self, id):
         """
         Release the file lock associated with this id
-        Note: This method must be called within the 'with' block
+        Note: This method must be called within the 'with' block and
+              before EVERY exit point of that 'with' block
         
         id -- int
         """

skypieamc/templates/derived/file/list.mako

     <% count = 0 %>
     %for f in c.paginator:
       <tr id="${f.id}" class="${'odd' if count % 2 else 'even'}">
-        <td class="file-info">
-          <dl>
-            <dt>
-              ${h.link_to(
-                  '{0}...'.format(f.name[:50]) if len(f.name) > 50 else f.name,
-                  h.url_for('#{0}'.format(f.id)),
-                  title=f.name
-                )}
-            </dt>
-            <dd>
-              <span class="mime-icon">${self.mime_icon(f.name)}</span>
-              Uploaded: ${f.uploaded.strftime('%Y/%m/%d')}
-              | Size: ${h.readable_filesize(f.size)}
-              | ${f.download_count} Downloads
-            </dd>
-          </dl>
-        </td>
-        <td>${self.play(f.id)}</td>
-        <td>${self.download(f.id)}</td>
+        <td class="file-info">${self.file_info(f)}</td>
+        <td>${self.play(f)}</td>
+        <td>${self.download(f)}</td>
       </tr>
       <% count += 1 %>
     %endfor
   <p>No files have been uploaded yet, ${h.link_to('add one', h.url_for(controller='file', action='upload'))}?</p>
 %endif
 
-<%def name="mime_icon(fname)">
-  <%! import os.path %>
-  <%
-    mime_types = {
-      'video': ['.avi', '.mp4', '.mkv', '.rmvb', '.flv', '.wmv'],
-      'audio': ['.mp3', '.wav', '.flac', '.ape', '.wma'],
-      'image': ['.bmp', '.gif', '.jpg', '.jpeg', '.png'],
-      'package': ['.bz2', '.gz', '.tar', '.zip', '.rar', '.7z'],
-      'file': None
-    }
-    ext = os.path.splitext(fname)[1].lower()
-    tname = None
-    for tname, types in mime_types.iteritems():
-      if types is not None and ext in types:
-        break
-    else:
-      tname = 'file'
-  %>
-  ${h.image(h.url_for('/images/icon_{0}.png'.format(tname)), None)}
+<%def name="file_info(mfile)">
+  <% fname = mfile.name.encode('utf-8') %>
+  <dl>
+    <dt>
+      ${h.link_to(
+          '{0}...'.format(
+            fname[:50]) if len(fname) > 50 else fname,
+            h.url_for('#{0}'.format(mfile.id)),
+            title=fname
+        )}
+    </dt>
+    <dd>
+      <span class="mime-icon">${self.mime_icon(mfile.name)}</span>
+      Uploaded: ${mfile.uploaded.strftime('%Y/%m/%d')}
+      | Size: ${h.readable_filesize(mfile.size)}
+      | ${mfile.download_count} Downloads
+    </dd>
+  </dl>
 </%def>
 
-<%def name="download(id)">
+<%def name="mime_icon(fname)">
+  ${h.image(h.url_for('/images/icon_{0}.png'.format(h.file_type(fname))), None)}
+</%def>
+
+<%def name="play(mfile)">
+  %if mfile.playable is not None:
+    ${self._create_button(
+        h.url_for(controller='file', action='play', id=mfile.id),
+        class_='play', 
+        label='Play'
+      )}
+  %endif
+</%def>
+
+<%def name="download(mfile)">
   ${self._create_button(
-      h.url_for(controller='file', action='download', id=id),
+      h.url_for(controller='file', action='download', id=mfile.id),
       class_='download', 
       label='Download'
     )}
 </%def>
 
-<%def name="play(id)">
-  ${self._create_button(
-      h.url_for(controller='file', action='play', id=id),
-      class_='play', 
-      label='Play'
-    )}
-</%def>
-
 <%def name="_create_button(url, class_, label)">
   <div class="${class_}">
     <a href="${url}">
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.