Source

cciw-website / fabfile.py

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
from collections import namedtuple
from datetime import datetime
from fabric.api import run, local, abort, env, put
from fabric.contrib import files
from fabric.contrib import console
from fabric.decorators import hosts, runs_once
from fabric.context_managers import cd, lcd, settings, hide
import os
import os.path
join = os.path.join
import sys

#  fabfile for deploying CCIW
#
# == Overview ==
#
# === Development ===
#
# You need a root directory to hold everything, and the following
# sub directories:
#
#  project/    - holds a checkout of this repository
#                i.e. fabfile.py and siblings live in that dir.
#
#  usermedia/  - corresponds to MEDIA_ROOT
#
#  secure_downloads/     - corresponds to SECUREDOWNLOAD_SERVE_ROOT
#
#  secure_downloads_src/ - corresponds to SECUREDOWNLOAD_SOURCE
#
# === Deployment ===
#
# There are two targets, STAGING and PRODUCTION, which live on the same
# server. They are almost identical, with these differences:
# - STAGING is on staging.cciw.co.uk
# - PRODUCTION is on www.cciw.co.uk
# - They have different databases
# - They have different apps on the webfaction server
#    - for the django project app
#    - for the static app
# - STAGING has SSL turned off.
#
# settings_priv.py and settings.py controls these things.
#
# In each target, we aim for atomic switching from one version to the next.
# This is not quite possible, but as much as possible the different versions
# are kept separate, preparing the new one completely before switching to it.
#
# To achieve this, new code is uploaded to a new 'dest_dir' which is timestamped,
# inside the 'src' dir in the cciw app directory.

# /home/cciw/webapps/cciw/         # PRODUCTION or
# /home/cciw/webapps/cciw_staging/ # STAGING
#    src/
#       src-2010-10-11_07-20-34/
#          env/                    # virtualenv dir
#          project/                # uploaded from local
#          static/                 # built once uploaded
#       current/                   # symlink to src-???

# At the same level as 'src-2010-10-11_07-20-34', there is a 'current' symlink
# which points to the most recent one. The apache instance looks at this (and
# the virtualenv dir inside it) to run the app.

# There is a webfaction app that points to src/current/static for serving static
# media. (One for production, one for staging). There is also a 'cciw_usermedia'
# app which is currently shared between production and staging. (This will only
# be a problem if usermedia needs to be re-organised).

# For speed, a new src-XXX dir is created by copying the 'current' one, and then
# using rsync and other updates. This is much faster than transferring
# everything and also rebuilding the virtualenv from scratch.

# When deploying, once the new directory is ready, the apache instance is
# stopped, the database is upgraded, and the 'current' symlink is switched. Then
# the apache instance is started.

# The information about this layout is unfortunately spread around a couple of
# places - this file and the settings file - because it is needed in both at
# different times.


env.hosts = ["cciw@cciw.co.uk"]

this_dir = os.path.dirname(os.path.abspath(__file__))
parent_dir = os.path.dirname(this_dir)
webapps_root = '/home/cciw/webapps'

# The path (relative to parent_dir) to where the project source code is stored:
project_dir = 'project'
usermedia_local = os.path.join(parent_dir, 'usermedia')
usermedia_production = os.path.join(webapps_root, 'cciw_usermedia')

class Target(object):
    """
    Represents a place where the project is deployed to.

    """
    def __init__(self, django_app='', dbname=''):
        self.django_app = django_app
        self.dbname = dbname

        self.webapp_root = join(webapps_root, self.django_app)
        # src_root - the root of all sources on this target.
        self.src_root = join(self.webapp_root, 'src')
        self.current_version = SrcVersion('current', join(self.src_root, 'current'))

    def make_version(self, label):
        return SrcVersion(label, join(self.src_root, "src-%s" % label))

class SrcVersion(object):
    """
    Represents a version of the project sources on a Target
    """
    def __init__(self, label, src_dir):
        self.label = label
        # src_dir - the root of all sources for this version
        self.src_dir = src_dir
        # venv_dir - note that _update_virtualenv assumes this relative layout
        # of the 'env' dir and the 'project' dir
        self.venv_dir = join(self.src_dir, 'env')
        # project_dir - where the CCIW project srcs are stored.
        self.project_dir = join(self.src_dir, project_dir)
        # static_dir - this is defined with way in settings.py
        self.static_dir = join(self.src_dir, 'static')

        self.additional_sys_paths = [project_dir]

STAGING = Target(
    django_app = "cciw_staging",
    dbname = "cciw_staging",
)
PRODUCTION = Target(
    django_app = "cciw",
    dbname = "cciw",
)


@runs_once
def ensure_dependencies():
    pass


def test():
    ensure_dependencies()
    local("./manage.py test cciwmain officers bookings --settings=cciw.settings_tests", capture=False)


def _prepare_deploy():
    ensure_dependencies()
    # test that we can do forwards and backwards migrations?
    # check that there are no outstanding changes.


def backup_database(target, version):
    fname = "%s-%s.db" % (target.dbname, version.label)
    run("dump_cciw_db.sh %s %s" % (target.dbname, fname))


def drop_local_db():
    with cd('/'):
        with settings(warn_only=True):
            local("sudo -u postgres psql -U postgres -d template1 -c 'DROP DATABASE cciw;'")

def create_local_db():
    cmds = """
CREATE DATABASE cciw;
CREATE USER cciw WITH PASSWORD 'foo';
GRANT ALL ON DATABASE cciw TO cciw;
"""
    with cd('/'):
        for c in cmds.strip().split("\n"):
            with settings(warn_only=True):
                local("sudo -u postgres psql -U postgres -d template1 -c \"%s\"" % c)


def run_venv(command, **kwargs):
    run("source %s/bin/activate" % env.venv + " && " + command, **kwargs)


def virtualenv(venv_dir):
    """
    Context manager that establishes a virtualenv to use,
    """
    return settings(venv=venv_dir)


def _update_symlink(target, version):
    if files.exists(target.current_version.src_dir):
        run("rm %s" % target.current_version.src_dir) # assumes symlink
    run("ln -s %s %s" % (version.src_dir, target.current_version.src_dir))


def _update_virtualenv(version):
    # Update virtualenv in new dir.
    with cd(version.src_dir):
        # We should already have a virtualenv, but it will need paths updating
        run("virtualenv --python=python2.7 env")
        # Need this to stop ~/lib/ dirs getting in:
        run("touch env/lib/python2.7/sitecustomize.py")
        with virtualenv(version.venv_dir):
            with cd(version.project_dir):
                run_venv("pip install -r requirements.txt")

        # Need to add project to path.
        pth_file = '\n'.join("../../../../" + n for n in version.additional_sys_paths)
        pth_name = "deps.pth"
        with open(pth_name, "w") as fd:
            fd.write(pth_file)
        put(pth_name, join(version.venv_dir, "lib/python2.7/site-packages"))
        os.unlink(pth_name)


def _stop_apache(target):
    run(join(target.webapp_root, "apache2/bin/stop"))


def _start_apache(target):
    run(join(target.webapp_root, "apache2/bin/start"))


def _restart_apache(target):
    with settings(warn_only=True):
        _stop_apache(target)
    _start_apache(target)


def rsync_dir(local_dir, dest_dir):
    # clean first
    with settings(warn_only=True):
        local("find -L %s -name '*.pyc' | xargs rm || true" % local_dir, capture=True)
    local("rsync -z -r -L --delete --exclude='_build' --exclude='.hg' --exclude='.git' --exclude='.svn' --delete-excluded %s/ cciw@cciw.co.uk:%s" % (local_dir, dest_dir), capture=False)


def _update_project_sources(target, version):
    # This also copies the virtualenv which is contained in the same folder,
    # which saves a lot of time with installing.

    run("mkdir -p %s" % version.src_dir)
    with cd(version.src_dir):
        if files.exists(target.current_version.project_dir + "/.hg"):
            # Clone local copy if we can
            run("hg clone %s project" % target.current_version.project_dir)
        else:
            run("hg clone ssh://hg@bitbucket.org/spookylukey/cciw-website project")

        with cd(version.project_dir):
            # We update to the version that is currently checked out locally,
            # because, at least for staging, it might not be the tip of default.
            current_rev = local("hg id -i", capture=True)
            run("hg pull ssh://hg@bitbucket.org/spookylukey/cciw-website")
            run("hg update -r %s" % current_rev.strip("+"))

        # Avoid recreating the virtualenv if we can
        if files.exists(target.current_version.venv_dir):
            run("cp -a -L %s %s" % (target.current_version.venv_dir,
                                    version.src_dir))

    # Also need to sync files that are not in main sources VCS repo.
    local("rsync cciw/settings_priv.py cciw@cciw.co.uk:%s/cciw/settings_priv.py" % version.project_dir)


def _copy_protected_downloads():
    # We currently don't need this to be separate for staging and production
    rsync_dir(join(parent_dir, "secure_downloads_src"),
              join(webapps_root, 'cciw_protected_downloads_src'))


def _build_static(version):
    # This always copies all files anyway, and we want to delete any unwanted
    # files, so we start from clean dir.
    run("rm -rf %s" % version.static_dir)

    with virtualenv(version.venv_dir):
        with cd(version.project_dir):
            run_venv("./manage.py collectstatic -v 0 --settings=cciw.settings --noinput")

    run("chmod -R ugo+r %s" % version.static_dir)


def _is_south_installed(target):
    cmd = """psql -d %s -U %s -h localhost -c "select tablename from pg_catalog.pg_tables where tablename='south_migrationhistory';" """ % (target.dbname, target.dbname)
    out = run(cmd)
    if 'south_migrationhistory' not in out:
        return False

    cmd2 = """psql -d %s -U %s -h localhost -c "select migration from south_migrationhistory where migration='0001_initial';" """ % (target.dbname, target.dbname)
    out2 = run(cmd2)
    if '0001_initial' not in out2:
        return False

    return True


def _install_south(target, version):
    # A one time task to be run after South has been first added
    with virtualenv(version.venv_dir):
        with cd(version.project_dir):
            run_venv("./manage.py syncdb --settings=cciw.settings")
            run_venv("./manage.py migrate --all 0001 --fake --settings=cciw.settings")


def _update_db(target, version):
    with virtualenv(version.venv_dir):
        with cd(version.project_dir):
            run_venv("./manage.py syncdb --settings=cciw.settings --noinput")
            run_venv("./manage.py migrate --all --settings=cciw.settings --noinput")


def _deploy(target, quick=False):
    # If 'quick=True', then it assumes all changes are small presentation
    # changes, with no database changes or Python code changes or server restart
    # needed.  (This depends on the assumption that HTML/CSS/js are not cached
    # in the webserver in any way).
    _prepare_deploy()

    label = datetime.now().strftime("%Y-%m-%d_%H.%M.%S")
    version = target.make_version(label)

    if quick:
        _update_project_sources(target, version)
        _copy_protected_downloads()
        _build_static(version)
        _update_symlink(target, version)
    else:
        _update_project_sources(target, version)
        _copy_protected_downloads()
        _update_virtualenv(version)
        _build_static(version)

        # Ideally, we:
        # 1) stop web server
        # 2) updated db
        # 3) rollback if unsuccessful.
        # 4) restart webserver

        # In practice, for this low traffic site it is better to keep website
        # going for as much time as possible, and cope with any small bugs that
        # come from mismatch of db and code.

        db_backup_name = backup_database(target, version)
        _update_db(target, version)
        _stop_apache(target)
        _update_symlink(target, version)
        _start_apache(target)


def _clean(target):
    """
    Misc clean-up tasks
    """
    # Remove old src versions.
    with cd(target.src_root):
        with hide("stdout"):
            currentlink = run("readlink current").split('/')[-1]
            otherlinks = set([x.strip() for x in run("ls src-* -1d").split("\n")])
        otherlinks.remove(currentlink)
        otherlinks = list(otherlinks)
        otherlinks.sort()
        otherlinks.reverse()

        # Leave the most recent previous version, delete the rest
        for d in otherlinks[1:]:
            run("rm -rf %s" % d)


def deploy_staging(quick=False):
    _deploy(STAGING, quick=quick)


def deploy_production(quick=False):
    with lcd(this_dir):
        if local("hg st", capture=True).strip() != "":
            if not console.confirm("Project dir is not clean, merge to live will fail. Continue anyway?", default=False):
                sys.exit()

    _deploy(PRODUCTION, quick=quick)
    #  Update 'live' branch so that we can switch to it easily if needed.
    with lcd(this_dir):
        local('hg update -r live && hg merge -r default && hg commit -m "Merged from default" && hg update -r default', capture=False)


def quick_deploy_staging():
    deploy_staging(quick=True)


def quick_deploy_production():
    deploy_production(quick=True)


def _test_remote(target):
    version = target.current_version
    with virtualenv(version.venv_dir):
        with cd(version.project_dir):
            run_venv("./manage.py test cciwmain officers --settings=cciw.settings_tests")


def stop_apache_production():
    _stop_apache(PRODUCTION)


def stop_apache_staging():
    _stop_apache(STAGING)


def start_apache_production():
    _start_apache(PRODUCTION)


def start_apache_staging():
    _start_apache(STAGING)


def restart_apache_production():
    _restart_apache(PRODUCTION)


def restart_apache_staging():
    _restart_apache(STAGING)


def clean_staging():
    _clean(STAGING)


def clean_production():
    _clean(PRODUCTION)


def test_staging():
    _test_remote(STAGING)


def test_production():
    _test_remote(PRODUCTION)


def upload_usermedia():
    local("rsync -z -r %s/ cciw@cciw.co.uk:%s" % (usermedia_local, usermedia_production), capture=False)


def backup_usermedia():
    local("rsync -z -r  cciw@cciw.co.uk:%s/ %s" % (usermedia_production, usermedia_local), capture=False)


# TODO:
#  - backup db task. This should be run only in production, and copies
#    files to Amazon S3 service.
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.