fabfile.py

   1 import os
   2
   3 import pkg_resources
   4
   5 from fabric.api import env, abort, task
   6 from fabric.context_managers import settings, hide, lcd
   7 from fabric.contrib.console import confirm
   8 from fabric.contrib.files import exists
   9 from fabric.operations import local, require, prompt
  10
  11 # Required dependencies
  12 #
  13 #    PIP_INSTALL - install packages from pip:  name:version
  14 #
  15 #    GIT_INSTALL - install packages from git:
  16 #
  17 #                    url - git url for checkouts
  18 #                    development - head to checkout for dev.
  19 #                                  Defaults to master
  20 #                    production - head to checkout for prod.
  21 #                                 Defaults to master.
  22 #                    symlink - directory to symlink into project.
  23 #                              Uses project name if blank
  24 # Packages from git are given preference for dev environment. PIP is given
  25 # preference for production environments
  26
  27 PIP_INSTALL = dict((r.project_name, str(r)) for r in
  28                    pkg_resources.parse_requirements(
  29                        open("requirements/prod.txt").read()
  30                    ))
  31
  32 GIT_INSTALL = {
  33     'django-object-permissions': {
  34         'url': 'git://git.osuosl.org/gitolite/django/'
  35                'django_object_permissions',
  36         'development': 'develop',
  37     },
  38     'django-object-log': {
  39         'url': 'git://git.osuosl.org/gitolite/django/'
  40                'django_object_log',
  41         'development': 'develop',
  42     },
  43     'twisted_vncauthproxy': {
  44         'url': 'git://git.osuosl.org/gitolite/ganeti/'
  45                'twisted_vncauthproxy',
  46         'development': 'develop',
  47     },
  48 }
  49
  50 DEV = 'development'
  51 PROD = 'production'
  52
  53 # default environment settings - override these in environment methods if you
  54 # wish to have an environment that functions differently
  55 env.doc_root = '.'
  56 env.remote = False
  57 env.environment = PROD
  58 env.verbose = False
  59 # List of stuff to include in the tarball, recursive.
  60 env.MANIFEST = [
  61     # Directories
  62     "deprecated",
  63     "django_test_tools",
  64     "ganeti_web",
  65     "locale",
  66     # Files
  67     "AUTHORS",
  68     "CHANGELOG",
  69     "COPYING",
  70     "LICENSE",
  71     "README",
  72     "UPGRADING",
  73     "__init__.py",
  74     "fabfile.py",
  75     "manage.py",
  76     "requirements",
  77     "search_sites.py",
  78     "settings.py.dist",
  79     "urls.py",
  80 ]
  81
  82
  83 @task
  84 def dev():
  85     """
  86     Configure development deployment.
  87     """
  88     env.environment = DEV
  89
  90
  91 @task
  92 def deploy():
  93     """
  94     Install all dependencies from git and pip.
  95     """
  96
  97     install_dependencies_pip()
  98     install_dependencies_git()
  99     novnc()
 100
 101
 102 @task
 103 def clean():
 104     """
 105     In a development environment, remove all installed packages and symlinks.
 106     """
 107     with lcd('%(doc_root)s' % env):
 108         gitcmd = 'git clean -%sdX -e \!settings.py'
 109         print('Files to be removed:')
 110         local(gitcmd % 'n')
 111         if confirm('Are you certain you would like to remove these files?'):
 112             local(gitcmd % 'f')
 113         else:
 114             abort('Aborting clean.')
 115
 116
 117 @task
 118 def update():
 119     """
 120     In a development environment, update all develop branches.
 121     """
 122
 123     if env.environment != DEV:
 124         raise Exception('must be in a development environment in order to'
 125                         'update develop branches.')
 126     else:
 127         with lcd('%(doc_root)s/dependencies' % env):
 128             for git_dir, opts in GIT_INSTALL.items():
 129                 env.git_repo = git_dir
 130                 if (_exists('%(doc_root)s/dependencies/%(git_repo)s' % env) and
 131                         'development' in opts and 'checkout' not in opts):
 132                     with lcd(git_dir):
 133                         print 'Updating git repo: %(git_repo)s' % env
 134                         local('git pull --ff')
 135
 136
 137 def _exists(path):
 138     """
 139     A helper function to determine whether a path exists.
 140
 141     This function does the right thing in both remote and local environments.
 142     """
 143
 144     if env.remote:
 145         return exists(path)
 146     else:
 147         return os.path.exists(path)
 148
 149
 150 def create_virtualenv(virtualenv='venv', force=False):
 151     """
 152     Create a virtualenv for pip installations.
 153
 154     By default, the environment will be placed in the document root. Pass a
 155     path to override the location.
 156
 157     If ``force`` is False, then the environment will not be recreated if it
 158     already exists.
 159     """
 160
 161     env.virtualenv = virtualenv if virtualenv else env.doc_root
 162
 163     with lcd(env.doc_root):
 164         if force or not _exists('%(virtualenv)s/lib' % env):
 165             # XXX does this actually create a new environment if one already
 166             # exists there?
 167             local('virtualenv %(virtualenv)s --distribute' % env)
 168
 169             # now lets make sure the virtual env has the the newest pip
 170             local(str(verbose_check()+'--upgrade pip') % env)
 171
 172
 173 def create_env():
 174     """
 175     Setup environment for git dependencies.
 176     """
 177
 178     with lcd(env.doc_root):
 179         if _exists('dependencies'):
 180             print 'dependencies directory exists already'
 181         else:
 182             local('mkdir dependencies')
 183
 184
 185 def verbose_check():
 186     """
 187     Default to quiet install when env.verbose is false
 188     """
 189     install_str = '%(virtualenv)s/bin/pip install '
 190     if not env.verbose:
 191         install_str += '-q '
 192
 193     return install_str
 194
 195
 196 def install_dependencies_pip():
 197     """
 198     Install all dependencies available from pip.
 199     """
 200
 201     create_virtualenv()
 202
 203     with lcd(env.doc_root):
 204         # Run the installation with pip, passing in our
 205         # requirements/prod.txt.
 206         local(str(verbose_check()+'-r requirements/prod.txt') % env)
 207
 208
 209 def install_dependencies_git():
 210     """
 211     Install all dependencies available from git.
 212     """
 213
 214     if env.environment != DEV:
 215         # If we can satisfy all of our dependencies from pip alone, then don't
 216         # bother running the git installation.
 217         if all(p in PIP_INSTALL for p in GIT_INSTALL):
 218             print 'No git repos to install! Yay!'
 219             return
 220
 221     create_env()
 222
 223     for name in (set(GIT_INSTALL) - set(PIP_INSTALL)):
 224         opts = GIT_INSTALL[name]
 225
 226         # check for required values
 227         if 'url' not in opts:
 228             raise Exception('missing required argument "url" '
 229                             'for git repo: %s' % name)
 230
 231         # set git head to check out
 232         if env.environment in opts:
 233             opts['head'] = opts[env.environment]
 234         elif env.environment == DEV and 'production' in opts:
 235             opts['head'] = opts['production']
 236         else:
 237             opts['head'] = 'master'
 238
 239         # clone repo
 240         with lcd('%(doc_root)s/dependencies' % env):
 241             env.git_repo = name
 242             env.git_url = opts['url']
 243             if not _exists('%(doc_root)s/dependencies/%(git_repo)s' % env):
 244                 local('git clone %(git_url)s %(git_repo)s' % env)
 245
 246             # create branch if not using master
 247             if opts['head'] != 'master':
 248                 with lcd(name):
 249                     local('git fetch')
 250
 251                     # Attempt to create a tracked branch and update it.
 252                     with settings(hide('warnings', 'stderr'), warn_only=True):
 253                         local('git checkout -t origin/%(head)s' % opts)
 254                         local('git pull')
 255
 256         # install to virtualenv using setup.py if it exists.  Some repos might
 257         # not have it and will need to be symlinked into the project
 258         if _exists('%(doc_root)s/dependencies/%(git_repo)s/setup.py' % env):
 259             with lcd(env.doc_root):
 260                 local(
 261                     str(verbose_check()+'-e dependencies/%(git_repo)s') % env
 262                 )
 263
 264         else:
 265             # else, configure and create symlink to git repo
 266             with lcd(env.doc_root):
 267                 if 'symlink' in opts:
 268                     env.symlink = opts['symlink']
 269                     env.symlink_path = '%(doc_root)s/dependencies/' \
 270                                        '%(git_repo)s/%(symlink)s' % env
 271                 else:
 272                     env.symlink = name
 273                     env.symlink_path = '%(doc_root)s/dependencies/' \
 274                                        '%(git_repo)s' % env
 275
 276                 with settings(hide('warnings', 'stderr'), warn_only=True):
 277                     local('ln -sf %(symlink_path)s %(doc_root)s' % env)
 278
 279
 280 def novnc():
 281     """
 282     Grab noVNC.
 283     """
 284
 285     if _exists("%(doc_root)s/noVNC" % env):
 286         return
 287
 288     # Grab the tarball, pass it through filters. Heavy abuse of the fact that
 289     # shell=True in local().
 290     with lcd(env.doc_root):
 291         # -L follows redirects.
 292         local("curl https://github.com/kanaka/noVNC/tarball/v0.3 -L | tar xz")
 293         # The glob replaces a git revision.
 294         local("mv kanaka-noVNC-*/ noVNC")
 295
 296
 297 @task
 298 def tarball():
 299     """
 300     Package a release tarball.
 301     """
 302
 303     tarball = prompt('tarball name', default='ganeti-webmgr.tar.gz')
 304     files = ['ganeti_webmgr/%s' % file for file in env.MANIFEST]
 305     files = ' '.join(files)
 306
 307     with lcd('..'):
 308         data = dict(
 309             tarball=tarball,
 310             files=files
 311         )
 312         local('tar zcf %(tarball)s %(files)s --exclude=*.pyc' % data)
 313         local('mv %(tarball)s ./ganeti_webmgr/' % data)
 314
 315
 316 def _uncomment(filen, com):
 317     args = dict(filen=filen, com=com)
 318     local('sed -i.bak -r -e "/%(com)s/ '
 319           's/^([[:space:]]*)#[[:space:]]?/\\1/g" %(filen)s' % args)
 320
 321
 322 def _comment(filen, com):
 323     args = dict(filen=filen, com=com)
 324     local('sed -i.bak -r -e "s/(%(com)s)/#\\1/g" %(filen)s' % args)
 325
 326
 327 def ldap(state="enable"):
 328     """
 329     Enable LDAP settings, and install packages
 330     Depends on: libldap2-dev, libsasl2-dev
 331     """
 332
 333     filename = 'settings.py' % env
 334     env.virtualenv = env.doc_root
 335
 336     with lcd(env.doc_root):
 337         if state == "enable":
 338             # Install and enable LDAP settings
 339             if not _exists('/usr/include/lber.h'):
 340                 abort("Make sure libldap-dev is installed before continuing")
 341             if not _exists('/usr/include/sasl'):
 342                 abort("Make sure libsasl2-dev is"
 343                       " installed before continuing")
 344             local('%(virtualenv)s/bin/pip install -qr'
 345                   ' requirements-ldap.txt' % env)
 346
 347             _uncomment(filename, 'from ldap_settings')
 348             _uncomment(filename, "'django_auth_ldap")
 349         elif state == "disable":
 350             # Disable LDAP settings and uninstall requirments
 351             local('%(virtualenv)s/bin/pip uninstall -qyr'
 352                   ' requirements-ldap.txt' % env)
 353
 354             _comment(filename, 'from ldap_settings')
 355             _comment(filename, "'django_auth_ldap")
 356
 357
 358 @task
 359 def v():
 360     """
 361     Enable verbose output in some commands
 362     """
 363     env.verbose = True