fabfile.py

   1 import os
   2
   3 import pkg_resources
   4
   5 from fabric.api import env, abort, task
   6 from fabric.context_managers import settings, hide, lcd
   7 from fabric.contrib.console import confirm
   8 from fabric.contrib.files import exists
   9 from fabric.operations import local, require, prompt
  10
  11 # Required dependencies
  12 #
  13 #    PIP_INSTALL - install packages from pip:  name:version
  14 #
  15 #    GIT_INSTALL - install packages from git:
  16 #
  17 #                    url - git url for checkouts
  18 #                    development - head to checkout for dev.
  19 #                                  Defaults to master
  20 #                    production - head to checkout for prod.
  21 #                                 Defaults to master.
  22 #                    symlink - directory to symlink into project.
  23 #                              Uses project name if blank
  24 # Packages from git are given preference for dev environment. PIP is given
  25 # preference for production environments
  26
  27 PIP_INSTALL = dict((r.project_name, str(r)) for r in
  28                    pkg_resources.parse_requirements(
  29                        open("requirements/prod.txt").read()
  30                    ))
  31
  32 GIT_INSTALL = {
  33     'django-object-permissions': {
  34         'url': 'git://git.osuosl.org/gitolite/django/'
  35                'django_object_permissions',
  36         'development': 'develop',
  37     },
  38     'django-object-log': {
  39         'url': 'git://git.osuosl.org/gitolite/django/'
  40                'django_object_log',
  41         'development': 'develop',
  42     },
  43     'twisted_vncauthproxy': {
  44         'url': 'git://git.osuosl.org/gitolite/ganeti/'
  45                'twisted_vncauthproxy',
  46         'development': 'develop',
  47     },
  48 }
  49
  50 DEV = 'development'
  51 PROD = 'production'
  52
  53 # default environment settings - override these in environment methods if you
  54 # wish to have an environment that functions differently
  55 env.doc_root = '.'
  56 env.remote = False
  57 env.environment = PROD
  58 env.verbose = False
  59 # List of stuff to include in the tarball, recursive.
  60 env.MANIFEST = [
  61     # Directories
  62     "deprecated",
  63     "django_test_tools",
  64     "docs/source",
  65     "docs/Makefile",
  66     "ganeti_web",
  67     "locale",
  68     "muddle",
  69     "muddle_users",
  70     # Files
  71     "AUTHORS",
  72     "CHANGELOG",
  73     "COPYING",
  74     "LICENSE",
  75     "LICENSE.muddle",
  76     "README.rst",
  77     "UPGRADING",
  78     "__init__.py",
  79     "fabfile.py",
  80     "manage.py",
  81     "requirements",
  82     "search_sites.py",
  83     "settings.py.dist",
  84     "ldap_settings.py.dist",
  85     "urls.py",
  86 ]
  87
  88
  89 @task
  90 def dev():
  91     """
  92     Configure development deployment.
  93     """
  94     env.environment = DEV
  95
  96
  97 @task
  98 def deploy():
  99     """
 100     Install all dependencies from git and pip.
 101     """
 102
 103     install_dependencies_pip()
 104     install_dependencies_git()
 105     novnc()
 106
 107
 108 @task
 109 def clean():
 110     """
 111     In a development environment, remove all installed packages and symlinks.
 112     """
 113     with lcd('%(doc_root)s' % env):
 114         gitcmd = 'git clean -%sdX -e \!settings.py'
 115         print('Files to be removed:')
 116         local(gitcmd % 'n')
 117         if confirm('Are you certain you would like to remove these files?'):
 118             local(gitcmd % 'f')
 119         else:
 120             abort('Aborting clean.')
 121
 122
 123 @task
 124 def update():
 125     """
 126     In a development environment, update all develop branches.
 127     """
 128
 129     if env.environment != DEV:
 130         raise Exception('must be in a development environment in order to'
 131                         'update develop branches.')
 132     else:
 133         with lcd('%(doc_root)s/dependencies' % env):
 134             for git_dir, opts in GIT_INSTALL.items():
 135                 env.git_repo = git_dir
 136                 if (_exists('%(doc_root)s/dependencies/%(git_repo)s' % env) and
 137                         'development' in opts and 'checkout' not in opts):
 138                     with lcd(git_dir):
 139                         print 'Updating git repo: %(git_repo)s' % env
 140                         local('git pull --ff')
 141
 142
 143 def _exists(path):
 144     """
 145     A helper function to determine whether a path exists.
 146
 147     This function does the right thing in both remote and local environments.
 148     """
 149
 150     if env.remote:
 151         return exists(path)
 152     else:
 153         return os.path.exists(path)
 154
 155
 156 def create_virtualenv(virtualenv='venv', force=False):
 157     """
 158     Create a virtualenv for pip installations.
 159
 160     By default, the environment will be placed in the document root. Pass a
 161     path to override the location.
 162
 163     If ``force`` is False, then the environment will not be recreated if it
 164     already exists.
 165     """
 166
 167     env.virtualenv = virtualenv if virtualenv else env.doc_root
 168
 169     with lcd(env.doc_root):
 170         if force or not _exists('%(virtualenv)s/lib' % env):
 171             # XXX does this actually create a new environment if one already
 172             # exists there?
 173             local('virtualenv %(virtualenv)s --distribute --no-site-packages' % env)
 174
 175             # now lets make sure the virtual env has the the newest pip
 176             local(str(verbose_check()+'--upgrade pip') % env)
 177
 178
 179 def create_env():
 180     """
 181     Setup environment for git dependencies.
 182     """
 183
 184     with lcd(env.doc_root):
 185         if _exists('dependencies'):
 186             print 'dependencies directory exists already'
 187         else:
 188             local('mkdir dependencies')
 189
 190
 191 def verbose_check():
 192     """
 193     Default to quiet install when env.verbose is false
 194     """
 195     install_str = '%(virtualenv)s/bin/pip install '
 196     if not env.verbose:
 197         install_str += '-q '
 198
 199     return install_str
 200
 201
 202 def install_dependencies_pip():
 203     """
 204     Install all dependencies available from pip.
 205     """
 206
 207     create_virtualenv()
 208
 209     with lcd(env.doc_root):
 210         # Run the installation with pip, passing in our
 211         # requirements/prod.txt.
 212         local(str(verbose_check()+'-r requirements/prod.txt') % env)
 213
 214
 215 def install_dependencies_git():
 216     """
 217     Install all dependencies available from git.
 218     """
 219
 220     if env.environment != DEV:
 221         # If we can satisfy all of our dependencies from pip alone, then don't
 222         # bother running the git installation.
 223         if all(p in PIP_INSTALL for p in GIT_INSTALL):
 224             print 'No git repos to install! Yay!'
 225             return
 226
 227     create_env()
 228
 229     for name in (set(GIT_INSTALL) - set(PIP_INSTALL)):
 230         opts = GIT_INSTALL[name]
 231
 232         # check for required values
 233         if 'url' not in opts:
 234             raise Exception('missing required argument "url" '
 235                             'for git repo: %s' % name)
 236
 237         # set git head to check out
 238         if env.environment in opts:
 239             opts['head'] = opts[env.environment]
 240         elif env.environment == DEV and 'production' in opts:
 241             opts['head'] = opts['production']
 242         else:
 243             opts['head'] = 'master'
 244
 245         # clone repo
 246         with lcd('%(doc_root)s/dependencies' % env):
 247             env.git_repo = name
 248             env.git_url = opts['url']
 249             if not _exists('%(doc_root)s/dependencies/%(git_repo)s' % env):
 250                 local('git clone %(git_url)s %(git_repo)s' % env)
 251
 252             # create branch if not using master
 253             if opts['head'] != 'master':
 254                 with lcd(name):
 255                     local('git fetch')
 256
 257                     # Attempt to create a tracked branch and update it.
 258                     with settings(hide('warnings', 'stderr'), warn_only=True):
 259                         local('git checkout -t origin/%(head)s' % opts)
 260                         local('git pull')
 261
 262         # install to virtualenv using setup.py if it exists.  Some repos might
 263         # not have it and will need to be symlinked into the project
 264         if _exists('%(doc_root)s/dependencies/%(git_repo)s/setup.py' % env):
 265             with lcd(env.doc_root):
 266                 local(
 267                     str(verbose_check()+'-e dependencies/%(git_repo)s') % env
 268                 )
 269
 270         else:
 271             # else, configure and create symlink to git repo
 272             with lcd(env.doc_root):
 273                 if 'symlink' in opts:
 274                     env.symlink = opts['symlink']
 275                     env.symlink_path = '%(doc_root)s/dependencies/' \
 276                                        '%(git_repo)s/%(symlink)s' % env
 277                 else:
 278                     env.symlink = name
 279                     env.symlink_path = '%(doc_root)s/dependencies/' \
 280                                        '%(git_repo)s' % env
 281
 282                 with settings(hide('warnings', 'stderr'), warn_only=True):
 283                     local('ln -sf %(symlink_path)s %(doc_root)s' % env)
 284
 285
 286 def novnc():
 287     """
 288     Grab noVNC.
 289     """
 290
 291     if _exists("%(doc_root)s/noVNC" % env):
 292         return
 293
 294     # Grab the tarball, pass it through filters. Heavy abuse of the fact that
 295     # shell=True in local().
 296     with lcd(env.doc_root):
 297         # -L follows redirects.
 298         local("curl https://github.com/kanaka/noVNC/tarball/v0.3 -L | tar xz")
 299         # The glob replaces a git revision.
 300         local("mv kanaka-noVNC-*/ noVNC")
 301
 302
 303 @task
 304 def tarball():
 305     """
 306     Package a release tarball.
 307     """
 308
 309     tarball = prompt('tarball name', default='ganeti-webmgr.tar.gz')
 310     files = ['ganeti_webmgr/%s' % file for file in env.MANIFEST]
 311     files = ' '.join(files)
 312
 313     with lcd('..'):
 314         data = dict(
 315             tarball=tarball,
 316             files=files
 317         )
 318         local('tar zcf %(tarball)s %(files)s --exclude=*.pyc' % data)
 319         local('mv %(tarball)s ./ganeti_webmgr/' % data)
 320
 321
 322 def _uncomment(filen, com):
 323     args = dict(filen=filen, com=com)
 324     local('sed -i.bak -r -e "/%(com)s/ '
 325           's/^([[:space:]]*)#[[:space:]]?/\\1/g" %(filen)s' % args)
 326
 327
 328 def _comment(filen, com):
 329     args = dict(filen=filen, com=com)
 330     local('sed -i.bak -r -e "s/(%(com)s)/#\\1/g" %(filen)s' % args)
 331
 332
 333 @task
 334 def ldap(state="enable", virtualenv='venv'):
 335     """
 336     Enable LDAP settings, and install packages
 337     Depends on: libldap2-dev, libsasl2-dev
 338     """
 339
 340     filename = 'settings.py'
 341     env.virtualenv = virtualenv if virtualenv else env.doc_root
 342
 343     with lcd(env.doc_root):
 344         if state == "enable":
 345             # Install and enable LDAP settings
 346             if not _exists('/usr/include/lber.h'):
 347                 abort("Make sure libldap-dev is installed before continuing")
 348             if not _exists('/usr/include/sasl'):
 349                 abort("Make sure libsasl2-dev is"
 350                       " installed before continuing")
 351             local(str(verbose_check()+'-r requirements/ldap.txt') % env)
 352
 353             _uncomment(filename, 'from ldap_settings')
 354             _uncomment(filename, "'django_auth_ldap")
 355         elif state == "disable":
 356             # Disable LDAP settings and uninstall requirments
 357             local('%(virtualenv)s/bin/pip uninstall '
 358                   '-r requirements/ldap.txt' % env)
 359
 360             _comment(filename, 'from ldap_settings')
 361             _comment(filename, "'django_auth_ldap")
 362
 363
 364 @task
 365 def v():
 366     """
 367     Enable verbose output in some commands
 368     """
 369     env.verbose = True