Mike Bayer avatar Mike Bayer committed 410898a

- The :class:`.ScriptDirectory` system that loads migration files
from a ``versions/`` directory now supports so-called
"sourceless" operation, where the ``.py`` files are not present
and instead ``.pyc`` or ``.pyo`` files are directly present where
the ``.py`` files should be. Note that while Python 3.3 has a
new system of locating ``.pyc``/``.pyo`` files within a directory
called ``__pycache__`` (e.g. PEP-3147), PEP-3147 maintains
support for the "source-less imports" use case, where the
``.pyc``/``.pyo`` are in present in the "old" location, e.g. next
to the ``.py`` file; this is the usage that's supported even when
running Python3.3. #163

Comments (0)

Files changed (8)

 *.pyc
+*.pyo
 build/
 dist/
 docs/build/output/

alembic/__init__.py

 from os import path
 
-__version__ = '0.6.2'
+__version__ = '0.6.3'
 
 package_dir = path.abspath(path.dirname(__file__))
 

alembic/compat.py

 
 if py33:
     from importlib import machinery
-    def load_module(module_id, path):
-        return machinery.SourceFileLoader(module_id, path).load_module()
+    def load_module_py(module_id, path):
+        return machinery.SourceFileLoader(module_id, path).load_module(module_id)
+
+    def load_module_pyc(module_id, path):
+        return machinery.SourcelessFileLoader(module_id, path).load_module(module_id)
+
 else:
     import imp
-    def load_module(module_id, path):
-        fp = open(path, 'rb')
-        try:
+    def load_module_py(module_id, path):
+        with open(path, 'rb') as fp:
             mod = imp.load_source(module_id, path, fp)
             if py2k:
                 source_encoding = parse_encoding(fp)
                 if source_encoding:
                     mod._alembic_source_encoding = source_encoding
             return mod
-        finally:
-            fp.close()
+
+    def load_module_pyc(module_id, path):
+        with open(path, 'rb') as fp:
+            mod = imp.load_compiled(module_id, path, fp)
+            # no source encoding here
+            return mod
 
 try:
     exec_ = getattr(compat_builtins, 'exec')

alembic/script.py

 import shutil
 from . import util
 
-_rev_file = re.compile(r'.*\.py$')
+_rev_file = re.compile(r'(.*\.py)(c|o)?$')
 _legacy_rev = re.compile(r'([a-f0-9]+)\.py$')
 _mod_def_re = re.compile(r'(upgrade|downgrade)_([a-z0-9]+)')
 _slug_re = re.compile(r'\w+')
 
     @classmethod
     def _from_filename(cls, dir_, filename):
-        if not _rev_file.match(filename):
+        py_match = _rev_file.match(filename)
+
+        if not py_match:
             return None
+
+        py_filename = py_match.group(1)
+        is_c = py_match.group(2) == 'c'
+        is_o = py_match.group(2) == 'o'
+
+        if is_o or is_c:
+            py_exists = os.path.exists(os.path.join(dir_, py_filename))
+            pyc_exists = os.path.exists(os.path.join(dir_, py_filename + "c"))
+
+            # prefer .py over .pyc because we'd like to get the
+            # source encoding; prefer .pyc over .pyo because we'd like to
+            # have the docstrings which a -OO file would not have
+            if py_exists or is_o and pyc_exists:
+                return None
+
         module = util.load_python_file(dir_, filename)
+
         if not hasattr(module, "revision"):
             # attempt to get the revision id from the script name,
             # this for legacy only
 from sqlalchemy.engine import url
 from sqlalchemy import __version__
 
-from .compat import callable, exec_, load_module, binary_type
+from .compat import callable, exec_, load_module_py, load_module_pyc, binary_type
 
 class CommandError(Exception):
     pass
 
     module_id = re.sub(r'\W', "_", filename)
     path = os.path.join(dir_, filename)
-    module = load_module(module_id, path)
+    _, ext = os.path.splitext(filename)
+    if ext == ".py":
+        module = load_module_py(module_id, path)
+    elif ext in (".pyc", ".pyo"):
+        module = load_module_pyc(module_id, path)
     del sys.modules[module_id]
     return module
 
+def simple_pyc_file_from_path(path):
+    if sys.flags.optimize:
+        return path + "o"  # e.g. .pyo
+    else:
+        return path + "c"  # e.g. .pyc
+
 def pyc_file_from_path(path):
     """Given a python source path, locate the .pyc.
 
     if has3147:
         return imp.cache_from_source(path)
     else:
-        return path + "c"
+        return simple_pyc_file_from_path(path)
 
 def rev_id():
     val = int(uuid.uuid4()) % 100000000000000

docs/build/changelog.rst

 ==========
 Changelog
 ==========
+.. changelog::
+    :version: 0.6.3
+
+    .. change::
+      :tags: feature
+      :tickets: 163
+
+     The :class:`.ScriptDirectory` system that loads migration files
+     from a  ``versions/`` directory now supports so-called
+     "sourceless" operation,  where the ``.py`` files are not present
+     and instead ``.pyc`` or ``.pyo`` files are directly present where
+     the ``.py`` files should be.  Note that while Python 3.3 has a
+     new system of locating ``.pyc``/``.pyo`` files within a directory
+     called ``__pycache__`` (e.g. PEP-3147), PEP-3147 maintains
+     support for the "source-less imports" use case, where the
+     ``.pyc``/``.pyo`` are in present in the "old" location, e.g. next
+     to the ``.py`` file; this is the usage that's supported even when
+     running Python3.3.
+
 
 .. changelog::
     :version: 0.6.2

tests/__init__.py

     shutil.rmtree(staging_directory, True)
 
 
-def write_script(scriptdir, rev_id, content, encoding='ascii'):
+def write_script(scriptdir, rev_id, content, encoding='ascii', sourceless=False):
     old = scriptdir._revision_map[rev_id]
     path = old.path
 
     scriptdir._revision_map[script.revision] = script
     script.nextrev = old.nextrev
 
+    if sourceless:
+        # note that if -O is set, you'd see pyo files here,
+        # the pyc util function looks at sys.flags.optimize to handle this
+        assert os.access(pyc_path, os.F_OK)
+        # look for a non-pep3147 path here.
+        # if not present, need to copy from __pycache__
+        simple_pyc_path = util.simple_pyc_file_from_path(path)
+        if not os.access(simple_pyc_path, os.F_OK):
+            shutil.copyfile(pyc_path, simple_pyc_path)
+        os.unlink(path)
+
 
 def three_rev_fixture(cfg):
     a = util.rev_id()

tests/test_versioning.py

     assert_raises_message
 
 class VersioningTest(unittest.TestCase):
+    sourceless = False
+
     def test_001_revisions(self):
         global a, b, c
         a = util.rev_id()
     def downgrade():
         op.execute("DROP TABLE foo")
 
-    """ % a)
+    """ % a, sourceless=self.sourceless)
 
         script.generate_revision(b, None, refresh=True)
         write_script(script, b, """
     def downgrade():
         op.execute("DROP TABLE bar")
 
-    """ % (b, a))
+    """ % (b, a), sourceless=self.sourceless)
 
         script.generate_revision(c, None, refresh=True)
         write_script(script, c, """
     def downgrade():
         op.execute("DROP TABLE bat")
 
-    """ % (c, b))
+    """ % (c, b), sourceless=self.sourceless)
 
 
     def test_002_upgrade(self):
 
         """)
 
+
+class SourcelessVersioningTest(VersioningTest):
+    sourceless = True
+
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.