Commits

Kirill Simonov committed e909649

Refactored the build script for source packages.

  • Participants
  • Parent commits 3dd08eb

Comments (0)

Files changed (4)

tool/data/pkg/source/moves.yaml

+#
+# Copyright (c) 2006-2012, Prometheus Research, LLC
+#
+
+#
+# This file specifies the layout of source packages.  The first
+# document indicates common files and attributes, the rest describe
+# individual source packages.
+#
+
+---
+code:
+files:
+  README:
+  LICENSE: [LICENSE-ALL, "", LICENSE-FREE-AGPL, "",
+            LICENSE-PERMISSIVE, "", LICENSE-EVALUATION]
+  setup.py: setup.py
+variables:
+  name:
+  version: # populated by the build script
+  description:
+  author: Clark C. Evans and Kirill Simonov; Prometheus Research, LLC
+  author-email: cce@clarkevans.com, xi@resolvent.net
+  license: AGPLv3 or Permissive for use with Open Source databases
+  keywords: sql relational database query language
+  platforms: Any
+  url: http://htsql.org/
+  classifiers:
+  package-dir: src
+  packages:
+  install-requires:
+  console-scripts:
+  htsql-addons: # populated by the build script
+  with-doc: false # not for substitution, used by the build script
+
+---
+code: htsql
+files:
+  README: README-CORE
+variables:
+  name: HTSQL
+  description: A Database Query Language (core & SQLite backend)
+  classifiers: |
+    Development Status :: 4 - Beta
+    Environment :: Console
+    Environment :: Web Environment
+    Intended Audience :: Developers
+    Intended Audience :: Information Technology
+    Intended Audience :: Science/Research
+    License :: OSI Approved :: GNU Affero General Public License v3
+    License :: Free To Use But Restricted
+    License :: Other/Proprietary License
+    Programming Language :: Python
+    Programming Language :: Python :: 2.5
+    Programming Language :: Python :: 2.6
+    Programming Language :: Python :: 2.7
+    Programming Language :: SQL
+    Topic :: Database :: Front-Ends
+    Topic :: Internet :: WWW/HTTP :: WSGI
+    Topic :: Software Development :: Libraries
+  packages: |
+    htsql
+    htsql_sqlite
+    sphinxcontrib
+  install-requires: |
+    setuptools
+    pyyaml
+  console-scripts: |
+    htsql-ctl = htsql.ctl:main
+  with-doc: true
+
+---
+code: htsql-pgsql
+files:
+  README: README-PGSQL
+variables:
+  name: HTSQL-PGSQL
+  description: A Database Query Language (PostgreSQL backend)
+  packages: |
+    htsql_pgsql
+  install-requires: |
+    HTSQL
+    psycopg2
+
+---
+code: htsql-mysql
+files:
+  README: README-MYSQL
+variables:
+  name: HTSQL-MYSQL
+  description: A Database Query Language (MySQL backend)
+  packages: |
+    htsql_mysql
+  install-requires: |
+    HTSQL
+    MySQL-python
+
+---
+code: htsql-oracle
+files:
+  README: README-ORACLE
+  LICENSE: LICENSE-EVALUATION
+variables:
+  name: HTSQL-ORACLE
+  description: A Database Query Language (Oracle backend)
+  license: Evaluation License
+  packages: |
+    htsql_oracle
+  install-requires: |
+    HTSQL
+    cx_Oracle
+
+---
+code: htsql-mssql
+files:
+  README: README-MSSQL
+  LICENSE: LICENSE-EVALUATION
+variables:
+  name: HTSQL-MSSQL
+  description: A Database Query Language (Microsoft SQL Server backend)
+  license: Evaluation License
+  packages: |
+    htsql_mssql
+  install-requires: |
+    HTSQL
+    pymssql
+

tool/data/pkg/source/setup.py

+#
+# Copyright (c) 2006-2012, Prometheus Research, LLC
+#
+
+
+#
+# To install ${NAME}, run `python setup.py install`.
+#
+
+
+from setuptools import setup
+import os.path
+
+
+NAME = "${NAME}"
+VERSION = "${VERSION}"
+DESCRIPTION = "${DESCRIPTION}"
+LONG_DESCRIPTION = open(os.path.join(os.path.dirname(__file__),
+                                     "README")).read()
+AUTHOR = "${AUTHOR}"
+AUTHOR_EMAIL = "${AUTHOR_EMAIL}"
+LICENSE = "${LICENSE}"
+KEYWORDS = "${KEYWORDS}"
+PLATFORMS = "${PLATFORMS}"
+URL = "${URL}"
+CLASSIFIERS = """
+${CLASSIFIERS}""".strip().splitlines() or None
+PACKAGE_DIR = {'': '${PACKAGE_DIR}'}
+INCLUDE_PACKAGE_DATA = True
+ZIP_SAFE = False
+PACKAGES = """
+${PACKAGES}""".strip().splitlines()
+INSTALL_REQUIRES = """
+${INSTALL_REQUIRES}""".strip().splitlines()
+CONSOLE_SCRIPTS = """
+${CONSOLE_SCRIPTS}""".strip().splitlines() or None
+HTSQL_ADDONS = """
+${HTSQL_ADDONS}""".strip().splitlines() or None
+ENTRY_POINTS = {}
+if CONSOLE_SCRIPTS:
+    ENTRY_POINTS['console_scripts'] = CONSOLE_SCRIPTS
+if HTSQL_ADDONS:
+    ENTRY_POINTS['htsql.addons'] = HTSQL_ADDONS
+
+
+setup(name=NAME,
+      version=VERSION,
+      description=DESCRIPTION,
+      long_description=LONG_DESCRIPTION,
+      author=AUTHOR,
+      author_email=AUTHOR_EMAIL,
+      license=LICENSE,
+      keywords=KEYWORDS,
+      platforms=PLATFORMS,
+      url=URL,
+      classifiers=CLASSIFIERS,
+      package_dir=PACKAGE_DIR,
+      include_package_data=INCLUDE_PACKAGE_DATA,
+      zip_safe=ZIP_SAFE,
+      packages=PACKAGES,
+      install_requires=INSTALL_REQUIRES,
+      entry_points=ENTRY_POINTS)
+
+

tool/data/pkg/source/setup.yaml

-#
-# Copyright (c) 2006-2012, Prometheus Research, LLC
-#
-
-#
-# The first document contains common attributes; the others describe
-# individual source packages.
-#
-
----
-author: Clark C. Evans and Kirill Simonov; Prometheus Research, LLC
-author-email: cce@clarkevans.com, xi@resolvent.net
-license: Free (AGPLv3) / Permissive for use with FLOSS databases / Evaluation
-keywords: 'sql http uri relational database query language'
-platform: Any
-url: http://htsql.org/
-package-dir: {'': 'src'}
-include-package-data: true
-zip-safe: false
-readme-file: null
-license-file: [LICENSE-ALL, LICENSE-FREE-AGPL, LICENSE-PERMISSIVE, LICENSE-EVALUATION]
-
----
-name: HTSQL
-description: A Database Query Language (core & SQLite backend)
-long-description: null
-classifiers:
-- 'Development Status :: 4 - Beta'
-- 'Environment :: Console'
-- 'Environment :: Web Environment'
-- 'Intended Audience :: Developers'
-- 'Intended Audience :: Information Technology'
-- 'Intended Audience :: Science/Research'
-- 'License :: OSI Approved :: GNU Affero General Public License v3'
-- 'License :: Free To Use But Restricted'
-- 'License :: Other/Proprietary License'
-- 'Programming Language :: Python'
-- 'Programming Language :: Python :: 2.5'
-- 'Programming Language :: Python :: 2.6'
-- 'Programming Language :: Python :: 2.7'
-- 'Programming Language :: SQL'
-- 'Topic :: Database :: Front-Ends'
-- 'Topic :: Internet :: WWW/HTTP :: WSGI'
-- 'Topic :: Software Development :: Libraries'
-packages: ['htsql', 'htsql_sqlite', 'sphinxcontrib']
-install-requires: ['setuptools', 'pyyaml']
-console-scripts: ['htsql-ctl = htsql.ctl:main']
-readme-file: [README-CORE]
-
----
-name: HTSQL-PGSQL
-description: A Database Query Language (PostgreSQL backend)
-packages: ['htsql_pgsql']
-install-requires: ['HTSQL', 'psycopg2']
-readme-file: [README-PGSQL]
-
----
-name: HTSQL-MYSQL
-description: A Database Query Language (MySQL backend)
-packages: ['htsql_mysql']
-install-requires: ['HTSQL', 'MySQL-python']
-readme-file: [README-MYSQL]
-
----
-name: HTSQL-ORACLE
-description: A Database Query Language (Oracle backend)
-license: Evaluation
-packages: ['htsql_oracle']
-install-requires: ['HTSQL', 'cx_Oracle']
-license-file: [LICENSE-EVALUATION]
-readme-file: [README-ORACLE]
-
----
-name: HTSQL-MSSQL
-description: A Database Query Language (Microsoft SQL Server backend)
-license: Evaluation
-packages: ['htsql_mssql']
-install-requires: ['HTSQL', 'pymssql']
-license-file: [LICENSE-EVALUATION]
-readme-file: [README-MSSQL]
-
 deb_vm = LinuxBenchVM('deb', 'debian', 22)
 
 
-def make_setup(name, version, description=None, long_description=None,
-               author=None, author_email=None, license=None, keywords=None,
-               platform=None, url=None, classifiers=None, package_dir=None,
-               packages=None, include_package_data=None, zip_safe=None,
-               install_requires=None, console_scripts=None, addons=None,
-               readme_file=None, license_file=None):
-    data = []
-    if readme_file:
-        readme_data = "\n".join(open(DATA_ROOT+"/pkg/source/"+filename)
-                                       .read().strip()+"\n\n"
-                                for filename in readme_file)
-        data.append(("README", readme_data))
-    if license_file:
-        license_data = "\n".join(open(DATA_ROOT+"/pkg/source/"+filename)
-                                        .read().strip()+"\n\n"
-                                 for filename in license_file)
-        data.append(("LICENSE", license_data))
-    stream = StringIO.StringIO()
-    stream.write("#\n")
-    stream.write("# This is a setup script for %s-%s.\n" % (name, version))
-    stream.write("# Type `python setup.py install` to install %s.\n" % name)
-    stream.write("# This file was generated automatically.\n")
-    stream.write("#\n")
-    stream.write("\n")
-    stream.write("from setuptools import setup\n")
-    stream.write("\n")
-    stream.write("setup(\n")
-    stream.write("    name=%r,\n" % name)
-    stream.write("    version=%r,\n" % version)
-    if description is not None:
-        stream.write("    description=%r,\n" % description)
-    if long_description is not None:
-        stream.write("    long_description=%r,\n" % long_description)
-    elif readme_file:
-        stream.write("    long_description=open('README').read(),\n")
-    if author is not None:
-        stream.write("    author=%r,\n" % author)
-    if author_email is not None:
-        stream.write("    author_email=%r,\n" % author_email)
-    if license is not None:
-        stream.write("    license=%r,\n" % license)
-    if keywords is not None:
-        stream.write("    keywords=%r,\n" % keywords)
-    if platform is not None:
-        stream.write("    platform=%r,\n" % platform)
-    if url is not None:
-        stream.write("    url=%r,\n" % url)
-    if classifiers is not None:
-        stream.write("    classifiers=%s,\n" % pprint.pformat(classifiers))
-    if package_dir is not None:
-        stream.write("    package_dir=%r,\n" % package_dir)
-    if packages is not None:
-        all_packages = [package for package in setuptools.find_packages('src')
-                        if package.split('.', 1)[0] in packages]
-        stream.write("    packages=%r,\n" % all_packages)
-    if include_package_data is not None:
-        stream.write("    include_package_data=%r,\n" % include_package_data)
-    if zip_safe is not None:
-        stream.write("    zip_safe=%r,\n" % zip_safe)
-    if install_requires is not None:
-        stream.write("    install_requires=%r,\n" % install_requires)
-    entry_points = {}
-    if console_scripts is not None:
-        entry_points['console_scripts'] = console_scripts
-    if addons is not None:
-        addons = [addon for addon in addons
-                  if addon.split('=', 1)[1].strip().split('.', 1)[0]
-                                                        in packages]
-        if addons:
-            entry_points['htsql.addons'] = addons
-    if entry_points:
-        stream.write("    entry_points=%s,\n" % pprint.pformat(entry_points))
-    stream.write(")\n")
-    stream.write("\n")
-    data.append(("setup.py", stream.getvalue()))
-    return data
+class Move(object):
+
+    def __init__(self, code, files, variables):
+        assert isinstance(code, str)
+        assert isinstance(files, dict)
+        for key in sorted(files):
+            assert isinstance(key, str)
+        assert isinstance(variables, dict)
+        for key in sorted(variables):
+            assert isinstance(key, str)
+        self.code = code
+        self.files = files
+        self.variables = variables
+
+    def __call__(self, src, dst):
+        assert os.path.isdir(src)
+        assert os.path.isdir(dst)
+        for filename in sorted(self.files):
+            parts = self.files[filename]
+            if parts is None:
+                parts = []
+            elif isinstance(parts, str):
+                parts = [parts]
+            assert isinstance(parts, list) and all(isinstance(part, str)
+                                                   for part in parts)
+            if not parts:
+                continue
+            filename = os.path.join(dst, filename)
+            dirname = os.path.dirname(filename)
+            if not os.path.exists(dirname):
+                os.makedirs(dirname)
+            stream = open(filename, 'w')
+            for part in parts:
+                if not part:
+                    stream.write("\n")
+                    continue
+                part = os.path.join(src, part)
+                assert os.path.isfile(part)
+                part = open(part).read()
+                part = self.substitute(part)
+                stream.write(part)
+
+    def substitute(self, data):
+        def replace(match):
+            key = match.group('key')
+            key = key.lower().replace('_', '-')
+            assert key in self.variables, key
+            value = self.variables[key]
+            assert value is None or isinstance(value, str), key
+            if value is None:
+                value = ""
+            return value
+        return re.sub(r"(?<!\$)\$\{(?P<key>[0-9a-zA-Z_-]+)\}", replace, data)
+
+    @classmethod
+    def load(cls, filename):
+        common_files = {}
+        common_variables = {}
+        moves = []
+        for document in yaml.load_all(open(filename)):
+            assert isinstance(document, dict)
+            assert all(key in ['code', 'files', 'variables']
+                       for key in sorted(document))
+            code = document.get('code')
+            files = document.get('files', {})
+            variables = document.get('variables', {})
+            if code is None:
+                common_files.update(files)
+                common_variables.update(variables)
+                continue
+            for key in common_files:
+                files.setdefault(key, common_files[key])
+            for key in common_variables:
+                variables.setdefault(key, common_variables[key])
+            move = cls(code, files, variables)
+            moves.append(move)
+        return moves
+
+load_moves = Move.load
 
 
 @job
         src_vm.stop()
     if os.path.exists("./build/pkg/src"):
         rmtree("./build/pkg/src")
+    if os.path.exists("./build/tmp"):
+        rmtree("./build/tmp")
     version = yaml.load(pipe_python("-c 'import setup, yaml;"
                                     " print yaml.dump(setup.get_version())'"))
-    addons = yaml.load(pipe_python("-c 'import setup, yaml;"
-                                   " print yaml.dump(setup.get_addons())'"))
-    cfg_all = list(yaml.load_all(open(DATA_ROOT+"/pkg/source/setup.yaml").read()))
-    with_doc = True
+    all_addons = yaml.load(pipe_python("-c 'import setup, yaml;"
+                                       " print yaml.dump(setup.get_addons())'"))
+    moves = load_moves(DATA_ROOT+"/pkg/source/moves.yaml")
     src_vm.start()
     try:
-        cfg_common = cfg_all[0]
-        for cfg in cfg_all[1:]:
-            for key in cfg_common:
-                if key not in cfg:
-                    cfg[key] = cfg_common[key]
-            for key, value in cfg.items():
-                if '-' in key:
-                    del cfg[key]
-                    key = key.replace('-', '_')
-                    cfg[key] = value
-            cfg['version'] = version
-            cfg['addons'] = addons
-            extra = make_setup(**cfg)
-            src_vm.forward(22)
-            run("hg clone --ssh='ssh -F %s' . ssh://linux-vm/htsql"
-                % (CTL_DIR+"/ssh_config"))
-            src_vm.unforward(22)
-            src_vm.run("cd htsql && hg update")
+        for move in moves:
+            with_doc = move.variables['with-doc']
+            packages = move.variables['packages'].strip().splitlines()
+            addons = "".join(addon+"\n" for addon in all_addons
+                             if addon.split('=', 1)[1].strip().split('.')[0]
+                                                                in packages)
+            move.variables['version'] = version
+            move.variables['htsql-addons'] = addons
+            mktree("./build/tmp")
+            run("hg archive ./build/tmp/htsql")
+            for dirname in sorted(glob.glob("./build/tmp/htsql/src/*")):
+                if os.path.basename(dirname) not in packages:
+                    rmtree(dirname)
+            packages = setuptools.find_packages("./build/tmp/htsql/src")
+            move.variables['packages'] = "".join(package+"\n"
+                                                 for package in packages)
+            if not with_doc:
+                rmtree("./build/tmp/htsql/doc")
+            move(DATA_ROOT+"/pkg/source", "./build/tmp/htsql")
+            src_vm.put("./build/tmp/htsql", ".")
             if with_doc:
                 src_vm.run("cd htsql &&"
                            " PYTHONPATH=src sphinx-build -d doc doc doc/html")
-                with_doc = False
-            for filename, data in extra:
-                src_vm.write("htsql/"+filename, data)
             src_vm.run("cd htsql && python setup.py sdist --formats=zip,gztar")
             if not os.path.exists("./build/pkg/src"):
                 mktree("./build/pkg/src")
             src_vm.get("./htsql/dist/*", "./build/pkg/src")
             src_vm.run("rm -rf htsql")
+            rmtree("./build/tmp")
     finally:
         src_vm.stop()
     log()
     if not (glob.glob("./build/pkg/src/HTSQL-*.tar.gz") and
             glob.glob("./build/pkg/src/HTSQL-*.zip")):
         raise fatal("cannot find source packages; run `job pkg-src` first")
-    if os.path.exists("./build/pkg/stage"):
-        rmtree("./build/pkg/stage")
-    mktree("./build/pkg/stage")
+    if os.path.exists("./build/tmp"):
+        rmtree("./build/tmp")
+    mktree("./build/tmp")
     archives = []
     for tgzname in sorted(glob.glob("./build/pkg/src/*.tar.gz")):
         dirname = tgzname[:-7]
         zipname = dirname+".zip"
         dirname = os.path.basename(dirname)
         project, version = dirname.rsplit('-', 1)
-        dirname = "./build/pkg/stage/"+dirname
-        run("tar -xzf %s -C ./build/pkg/stage" % tgzname)
+        dirname = "./build/tmp/"+dirname
+        run("tar -xzf %s -C ./build/tmp" % tgzname)
         mktree(dirname+"/dist")
         cp(tgzname, dirname+"/dist")
         cp(zipname, dirname+"/dist")
                  " register upload --sign --identity="+KEYSIG,
                  cd=dirname)
         archives.append((project, version))
-    rmtree("./build/pkg/stage")
+    rmtree("./build/tmp")
     log()
     log("Source distribution archives are uploaded to:")
     for project, version in archives: