Commits

Jun Omae committed b491ab6

0.13dev: New `AttachmentSetup` component migrates from `$ENV/attachmets` to `$ENV/files/attachments`. Each attachment is stored using SHA-1 of parent_id and filename.

Comments (0)

Files changed (2)

trac/attachment.py

 
 from cStringIO import StringIO
 from datetime import datetime
+import errno
 import os.path
 import re
 import shutil
 from trac.perm import PermissionError, IPermissionPolicy
 from trac.resource import *
 from trac.search import search_to_sql, shorten_result
-from trac.util import content_disposition, create_unique_file, get_reporter_id
+from trac.util import content_disposition, get_reporter_id
+from trac.util.compat import sha1
 from trac.util.datefmt import format_datetime, from_utimestamp, \
                               to_datetime, to_utimestamp, utc
 from trac.util.text import exception_to_unicode, pretty_size, print_table, \
                                    _('Invalid Attachment'))
 
     def _get_path(self, parent_realm, parent_id, filename):
+        path = os.path.join(self.env.path, 'files', 'attachments',
+                            parent_realm)
+        hash = sha1(parent_id.encode('utf-8')).hexdigest()
+        path = os.path.join(path, hash[0:3], hash)
+        if filename:
+            path = os.path.join(path, self._get_hashed_filename(filename))
+        return os.path.normpath(path)
+
+    def _get_hashed_filename(self, filename):
+        hash = sha1(filename.encode('utf-8')).hexdigest()
+        parts = os.path.splitext(filename)
+        return hash + parts[1] if parts[1] else hash
+    
+    def _get_path_old(self, parent_realm, parent_id, filename):
         path = os.path.join(self.env.path, 'attachments', parent_realm,
                             unicode_quote(parent_id))
         if filename:
             db("""
                 DELETE FROM attachment WHERE type=%s AND id=%s AND filename=%s
                 """, (self.parent_realm, self.parent_id, self.filename))
-            if os.path.isfile(self.path):
+            path = self.path
+            if os.path.isfile(path):
                 try:
-                    os.unlink(self.path)
+                    os.unlink(path)
                 except OSError, e:
                     self.env.log.error("Failed to delete attachment "
                                        "file %s: %s",
-                                       self.path,
+                                       path,
                                        exception_to_unicode(e, traceback=True))
                     raise TracError(_("Could not delete attachment"))
 
             dirname = os.path.dirname(new_path)
             if not os.path.exists(dirname):
                 os.makedirs(dirname)
-            if os.path.isfile(self.path):
+            path = self.path
+            if os.path.isfile(path):
                 try:
-                    os.rename(self.path, new_path)
+                    os.rename(path, new_path)
                 except OSError, e:
                     self.env.log.error("Failed to move attachment file %s: %s",
-                                       self.path,
+                                       path,
                                        exception_to_unicode(e, traceback=True))
                     raise TracError(_("Could not reparent attachment %(name)s",
                                       name=self.filename))
         # Make sure the path to the attachment is inside the environment
         # attachments directory
         attachments_dir = os.path.join(os.path.normpath(self.env.path),
-                                       'attachments')
-        commonprefix = os.path.commonprefix([attachments_dir, self.path])
+                                       'files', 'attachments')
+        dir = self.path
+        commonprefix = os.path.commonprefix([attachments_dir, dir])
         assert commonprefix == attachments_dir
 
-        if not os.access(self.path, os.F_OK):
-            os.makedirs(self.path)
-        filename = unicode_quote(filename)
-        path, targetfile = create_unique_file(os.path.join(self.path,
-                                                           filename))
+        if not os.access(dir, os.F_OK):
+            os.makedirs(dir)
+        filename, targetfile = self._create_unique_file(dir, filename)
         with targetfile:
-            # Note: `path` is an unicode string because `self.path` was one.
-            # As it contains only quoted chars and numbers, we can use `ascii`
-            basename = os.path.basename(path).encode('ascii')
-            filename = unicode_unquote(basename)
-
             with self.env.db_transaction as db:
                 db("INSERT INTO attachment VALUES (%s,%s,%s,%s,%s,%s,%s,%s)",
                    (self.parent_realm, self.parent_id, filename, self.size,
                     attachment_dir, exception_to_unicode(e, traceback=True))
             
     def open(self):
-        self.env.log.debug('Trying to open attachment at %s', self.path)
+        path = self.path
+        self.env.log.debug('Trying to open attachment at %s', path)
         try:
-            fd = open(self.path, 'rb')
+            fd = open(path, 'rb')
         except IOError:
             raise ResourceNotFound(_("Attachment '%(filename)s' not found",
                                      filename=self.filename))
         return fd
 
+    def _create_unique_file(self, dir, filename):
+        parts = os.path.splitext(filename)
+        flags = os.O_CREAT + os.O_WRONLY + os.O_EXCL
+        if hasattr(os, 'O_BINARY'):
+            flags += os.O_BINARY
+        idx = 1
+        while 1:
+            path = os.path.join(dir, self._get_hashed_filename(filename))
+            try:
+                return filename, os.fdopen(os.open(path, flags, 0666), 'w')
+            except OSError, e:
+                if e.errno != errno.EEXIST:
+                    raise
+                idx += 1
+                # A sanity check
+                if idx > 100:
+                    raise Exception('Failed to create unique name: ' + path)
+                filename = '%s.%d%s' % (parts[0], idx, parts[1])
+
+
+class AttachmentSetup(Component):
+
+    implements(IEnvironmentSetupParticipant)
+
+    required = True
+
+    # IEnvironmentSetupParticipant methods
+
+    def environment_created(self):
+        """Create the attachments directory."""
+        path = self.env.path
+        if path:
+            os.makedirs(os.path.join(path, 'files', 'attachments'))
+
+    def environment_needs_upgrade(self, db):
+        path = self.env.path
+        if path:
+            old_dir = os.path.join(path, 'attachments')
+            new_dir = os.path.join(path, 'files', 'attachments')
+            return os.path.exists(old_dir) and not os.path.exists(new_dir)
+
+    def upgrade_environment(self, db):
+        """Migrate attachments from old-style directory to new-style
+        directory.
+        """
+        path = self.env.path
+        os.makedirs(os.path.join(path, 'files', 'attachments'))
+
+        cursor = db.cursor()
+        cursor.execute("SELECT type, id, filename, description, size, time, "
+                       "author, ipnr FROM attachment ORDER BY type, id")
+        for row in cursor:
+            attachment = Attachment(self.env, row[0], row[1])
+            attachment._from_database(*row[2:])
+            self._copy_attachment_file(attachment)
+
+        shutil.rmtree(os.path.join(path, 'attachments'))
+
+    def _copy_attachment_file(self, attachment):
+        old_path = attachment._get_path_old(attachment.parent_realm,
+                                            attachment.parent_id,
+                                            attachment.filename)
+        if os.path.isfile(old_path):
+            path = attachment.path
+            dir = os.path.dirname(path)
+            if not os.path.exists(dir):
+                os.makedirs(dir)
+            shutil.copyfile(old_path, path)
+            shutil.copystat(old_path, path)
+
 
 class AttachmentModule(Component):
 
-    implements(IEnvironmentSetupParticipant, IRequestHandler,
-               INavigationContributor, IWikiSyntaxProvider,
+    implements(IRequestHandler, INavigationContributor, IWikiSyntaxProvider,
                IResourceManager)
 
     change_listeners = ExtensionPoint(IAttachmentChangeListener)
         For public sites where anonymous users can create attachments it is
         recommended to leave this option disabled (which is the default).""")
 
-    # IEnvironmentSetupParticipant methods
-
-    def environment_created(self):
-        """Create the attachments directory."""
-        if self.env.path:
-            os.mkdir(os.path.join(self.env.path, 'attachments'))
-
-    def environment_needs_upgrade(self, db):
-        return False
-
-    def upgrade_environment(self, db):
-        pass
-
     # INavigationContributor methods
 
     def get_active_navigation_item(self, req):

trac/tests/attachment.py

 from trac.test import EnvironmentStub
 
 
+hashes = {
+    '42': '92cfceb39d57d914ed8b14d0e37643de0797ae56',
+    'SomePage': 'd7e80bae461ca8568e794792f5520b603f540e06',
+    'Teh bar.jpg': 'ed9102c4aa099e92baf1073f824d21c5e4be5944',
+    'Teh foo.txt': 'ab97ba98d98fcf72b92e33a66b07077010171f70',
+    'bar.jpg': 'ae0faa593abf2b6f8871f6f32fe5b28d1c6572be',
+    'foo.txt': '9206ac42b532ef8e983470c251f4e1a365fd636c',
+    'foo.2.txt': 'a8fcfcc2ef4e400ee09ae53c1aabd7f5a5fda0c7',
+    u'ÜberSicht': 'a16c6837f6d3d2cc3addd68976db1c55deb694c8',
+}
+
+
 class TicketOnlyViewsTicket(Component):
     implements(IPermissionPolicy)
 
         self.env = EnvironmentStub()
         self.env.path = os.path.join(tempfile.gettempdir(), 'trac-tempenv')
         os.mkdir(self.env.path)
-        self.attachments_dir = os.path.join(self.env.path, 'attachments')
+        self.attachments_dir = os.path.join(self.env.path, 'files',
+                                            'attachments')
         self.env.config.set('trac', 'permission_policies',
                             'TicketOnlyViewsTicket, LegacyAttachmentPolicy')
         self.env.config.set('attachment', 'max_size', 512)
     def test_get_path(self):
         attachment = Attachment(self.env, 'ticket', 42)
         attachment.filename = 'foo.txt'
-        self.assertEqual(os.path.join(self.attachments_dir, 'ticket', '42',
-                                      'foo.txt'),
+        self.assertEqual(os.path.join(self.attachments_dir, 'ticket',
+                                      hashes['42'][0:3], hashes['42'],
+                                      hashes['foo.txt'] + '.txt'),
                          attachment.path)
         attachment = Attachment(self.env, 'wiki', 'SomePage')
         attachment.filename = 'bar.jpg'
-        self.assertEqual(os.path.join(self.attachments_dir, 'wiki', 'SomePage',
-                                      'bar.jpg'),
+        self.assertEqual(os.path.join(self.attachments_dir, 'wiki',
+                                      hashes['SomePage'][0:3],
+                                      hashes['SomePage'],
+                                      hashes['bar.jpg'] + '.jpg'),
                          attachment.path)
 
     def test_get_path_encoded(self):
         attachment = Attachment(self.env, 'ticket', 42)
         attachment.filename = 'Teh foo.txt'
-        self.assertEqual(os.path.join(self.attachments_dir, 'ticket', '42',
-                                      'Teh%20foo.txt'),
+        self.assertEqual(os.path.join(self.attachments_dir, 'ticket',
+                                      hashes['42'][0:3], hashes['42'],
+                                      hashes['Teh foo.txt'] + '.txt'),
                          attachment.path)
         attachment = Attachment(self.env, 'wiki', u'ÜberSicht')
         attachment.filename = 'Teh bar.jpg'
         self.assertEqual(os.path.join(self.attachments_dir, 'wiki',
-                                      '%C3%9CberSicht', 'Teh%20bar.jpg'),
+                                      hashes[u'ÜberSicht'][0:3],
+                                      hashes[u'ÜberSicht'],
+                                      hashes['Teh bar.jpg'] + '.jpg'),
                          attachment.path)
 
     def test_select_empty(self):
         attachment = Attachment(self.env, 'ticket', 42)
         attachment.insert('foo.txt', StringIO(''), 0)
         self.assertEqual('foo.2.txt', attachment.filename)
+        self.assertEqual(os.path.join(self.attachments_dir, 'ticket',
+                                      hashes['42'][0:3], hashes['42'],
+                                      hashes['foo.2.txt'] + '.txt'),
+                         attachment.path)
+        self.assert_(os.path.exists(attachment.path))
 
     def test_insert_outside_attachments_dir(self):
         attachment = Attachment(self.env, '../../../../../sth/private', 42)