ganeti_web/models.py

   1 # coding: utf-8
   2
   3 # Copyright (C) 2010 Oregon State University et al.
   4 # Copyright (C) 2010 Greek Research and Technology Network
   5 #
   6 # This program is free software; you can redistribute it and/or
   7 # modify it under the terms of the GNU General Public License
   8 # as published by the Free Software Foundation; either version 2
   9 # of the License, or (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
  19 # USA.
  20
  21 import binascii
  22 import cPickle
  23 from datetime import datetime, timedelta
  24 from hashlib import sha1
  25 import random
  26 import re
  27 import string
  28 import sys
  29 import time
  30
  31 from django.conf import settings
  32
  33 from django.contrib.auth.models import User, Group
  34 from django.contrib.contenttypes.generic import GenericForeignKey
  35 from django.contrib.contenttypes.models import ContentType
  36 from django.contrib.sites import models as sites_app
  37 from django.contrib.sites.management import create_default_site
  38 from django.core.validators import RegexValidator, MinValueValidator
  39 from django.db import models
  40 from django.db.models import BooleanField, Q, Sum
  41 from django.db.models.query import QuerySet
  42 from django.db.models.signals import post_save, post_syncdb
  43 from django.db.utils import DatabaseError
  44 from django.utils.encoding import force_unicode
  45 from django.utils.translation import ugettext_lazy as _
  46
  47 from django_fields.fields import PickleField
  48
  49 from ganeti_web.logs import register_log_actions
  50
  51 from object_log.models import LogItem
  52 log_action = LogItem.objects.log_action
  53
  54 from object_permissions.registration import register
  55
  56 from muddle_users import signals as muddle_user_signals
  57
  58 from ganeti_web import constants, management, permissions
  59 from ganeti_web.fields import (PatchedEncryptedCharField,
  60                                PreciseDateTimeField, SumIf)
  61 from ganeti_web.util import client
  62 from ganeti_web.util.client import GanetiApiError, REPLACE_DISK_AUTO
  63
  64 from south.signals import post_migrate
  65
  66 if settings.VNC_PROXY:
  67     from ganeti_web.util.vncdaemon.vapclient import (request_forwarding,
  68                                                      request_ssh)
  69
  70
  71 class QuerySetManager(models.Manager):
  72     """
  73     Useful if you want to define manager methods that need to chain. In this
  74     case create a QuerySet class within your model and add all of your methods
  75     directly to the queryset. Example:
  76
  77     class Foo(models.Model):
  78         enabled = fields.BooleanField()
  79         dirty = fields.BooleanField()
  80
  81         class QuerySet:
  82             def active(self):
  83                 return self.filter(enabled=True)
  84             def clean(self):
  85                 return self.filter(dirty=False)
  86
  87     Foo.objects.active().clean()
  88     """
  89
  90     def __getattr__(self, name, *args):
  91         # Cull under/dunder names to avoid certain kinds of recursion. Django
  92         # isn't super-bright here.
  93         if name.startswith('_'):
  94             raise AttributeError
  95         return getattr(self.get_query_set(), name, *args)
  96
  97     def get_query_set(self):
  98         return self.model.QuerySet(self.model)
  99
 100
 101 def generate_random_password(length=12):
 102     "Generate random sequence of specified length"
 103     return "".join(random.sample(string.letters + string.digits, length))
 104
 105 FINISHED_JOBS = 'success', 'unknown', 'error'
 106
 107 RAPI_CACHE = {}
 108 RAPI_CACHE_HASHES = {}
 109
 110
 111 def get_rapi(hash, cluster):
 112     """
 113     Retrieves the cached Ganeti RAPI client for a given hash.  The Hash is
 114     derived from the connection credentials required for a cluster.  If the
 115     client is not yet cached, it will be created and added.
 116
 117     If a hash does not correspond to any cluster then Cluster.DoesNotExist will
 118     be raised.
 119
 120     @param cluster - either a cluster object, or ID of object.  This is used
 121     for resolving the cluster if the client is not already found.  The id is
 122     used rather than the hash, because the hash is mutable.
 123
 124     @return a Ganeti RAPI client.
 125     """
 126     if hash in RAPI_CACHE:
 127         return RAPI_CACHE[hash]
 128
 129     # always look up the instance, even if we were given a Cluster instance
 130     # it ensures we are retrieving the latest credentials.  This helps avoid
 131     # stale credentials.  Retrieve only the values because we don't actually
 132     # need another Cluster instance here.
 133     if isinstance(cluster, (Cluster,)):
 134         cluster = cluster.id
 135     (credentials,) = Cluster.objects.filter(id=cluster) \
 136         .values_list('hash', 'hostname', 'port', 'username', 'password')
 137     hash, host, port, user, password = credentials
 138     user = user or None
 139     # decrypt password
 140     # XXX django-fields only stores str, convert to None if needed
 141     password = Cluster.decrypt_password(password) if password else None
 142     password = None if password in ('None', '') else password
 143
 144     # now that we know hash is fresh, check cache again. The original hash
 145     # could have been stale. This avoids constructing a new RAPI that already
 146     # exists.
 147     if hash in RAPI_CACHE:
 148         return RAPI_CACHE[hash]
 149
 150     # delete any old version of the client that was cached.
 151     if cluster in RAPI_CACHE_HASHES:
 152         del RAPI_CACHE[RAPI_CACHE_HASHES[cluster]]
 153
 154     # Set connect timeout in settings.py so that you do not learn patience.
 155     rapi = client.GanetiRapiClient(host, port, user, password,
 156                                    timeout=settings.RAPI_CONNECT_TIMEOUT)
 157     RAPI_CACHE[hash] = rapi
 158     RAPI_CACHE_HASHES[cluster] = hash
 159     return rapi
 160
 161
 162 def clear_rapi_cache():
 163     """
 164     clears the rapi cache
 165     """
 166     RAPI_CACHE.clear()
 167     RAPI_CACHE_HASHES.clear()
 168
 169
 170 ssh_public_key_re = re.compile(
 171     r'^ssh-(rsa|dsa|dss) [A-Z0-9+/=]+ .+$', re.IGNORECASE)
 172 ssh_public_key_error = _("Enter a valid RSA or DSA SSH key.")
 173 validate_sshkey = RegexValidator(ssh_public_key_re, ssh_public_key_error,
 174                                  "invalid")
 175
 176
 177 class CachedClusterObject(models.Model):
 178     """
 179     Parent class for objects which belong to Ganeti but have cached data in
 180     GWM.
 181
 182     The main point of this class is to permit saving lots of data from Ganeti
 183     so that we don't have to look things up constantly. The Ganeti RAPI is
 184     slow, so avoiding it as much as possible is a good idea.
 185
 186     This class provides transparent caching for all of the data that it
 187     serializes; no explicit cache accesses are required.
 188
 189     This model is abstract and may not be instantiated on its own.
 190     """
 191
 192     serialized_info = models.TextField(default="", editable=False)
 193     mtime = PreciseDateTimeField(null=True, editable=False)
 194     cached = PreciseDateTimeField(null=True, editable=False)
 195     ignore_cache = models.BooleanField(default=False)
 196
 197     last_job_id = None
 198     __info = None
 199     error = None
 200     ctime = None
 201     deleted = False
 202
 203     class Meta:
 204         abstract = True
 205
 206     def save(self, *args, **kwargs):
 207         """
 208         overridden to ensure info is serialized prior to save
 209         """
 210         if not self.serialized_info:
 211             self.serialized_info = cPickle.dumps(self.__info)
 212         super(CachedClusterObject, self).save(*args, **kwargs)
 213
 214     def __init__(self, *args, **kwargs):
 215         super(CachedClusterObject, self).__init__(*args, **kwargs)
 216         self.load_info()
 217
 218     @property
 219     def info(self):
 220         """
 221         A dictionary of metadata for this object.
 222
 223         This is a proxy for the ``serialized_info`` field. Reads from this
 224         property lazily access the field, and writes to this property will be
 225         lazily saved.
 226
 227         Writes to this property do *not* force serialization.
 228         """
 229
 230         if self.__info is None:
 231             if self.serialized_info:
 232                 self.__info = cPickle.loads(str(self.serialized_info))
 233         return self.__info
 234
 235     def _set_info(self, value):
 236         self.__info = value
 237         if value is not None:
 238             self.parse_info()
 239             self.serialized_info = ""
 240
 241     info = info.setter(_set_info)
 242
 243     def load_info(self):
 244         """
 245         Load cached info retrieved from the ganeti cluster.  This function
 246         includes a lazy cache mechanism that uses a timer to decide whether or
 247         not to refresh the cached information with new information from the
 248         ganeti cluster.
 249
 250         This will ignore the cache when self.ignore_cache is True
 251         """
 252
 253         epsilon = timedelta(0, 0, 0, settings.LAZY_CACHE_REFRESH)
 254
 255         if self.id:
 256             if (self.ignore_cache
 257                     or self.cached is None
 258                     or datetime.now() > self.cached + epsilon):
 259                 self.refresh()
 260             elif self.info:
 261                 self.parse_transient_info()
 262             else:
 263                 self.error = 'No Cached Info'
 264
 265     def parse_info(self):
 266         """
 267         Parse all of the attached metadata, and attach it to this object.
 268         """
 269
 270         self.parse_transient_info()
 271         data = self.parse_persistent_info(self.info)
 272         for k in data:
 273             setattr(self, k, data[k])
 274
 275     def refresh(self):
 276         """
 277         Retrieve and parse info from the ganeti cluster.  If successfully
 278         retrieved and parsed, this method will also call save().
 279
 280         If communication with Ganeti fails, an error will be stored in
 281         ``error``.
 282         """
 283
 284         job_data = self.check_job_status()
 285         for k, v in job_data.items():
 286             setattr(self, k, v)
 287
 288         # XXX this try/except is far too big; see if we can pare it down.
 289         try:
 290             info_ = self._refresh()
 291             if info_:
 292                 if info_['mtime']:
 293                     mtime = datetime.fromtimestamp(info_['mtime'])
 294                 else:
 295                     mtime = None
 296                 self.cached = datetime.now()
 297             else:
 298                 # no info retrieved, use current mtime
 299                 mtime = self.mtime
 300
 301             if self.id and (self.mtime is None or mtime > self.mtime):
 302                 # there was an update. Set info and save the object
 303                 self.info = info_
 304                 self.save()
 305             else:
 306                 # There was no change on the server. Only update the cache
 307                 # time. This bypasses the info serialization mechanism and
 308                 # uses a smaller query.
 309                 if job_data:
 310                     self.__class__.objects.filter(pk=self.id) \
 311                         .update(cached=self.cached, **job_data)
 312                 elif self.id is not None:
 313                     self.__class__.objects.filter(pk=self.id) \
 314                         .update(cached=self.cached)
 315
 316         except GanetiApiError, e:
 317             # Use regular expressions to match the quoted message
 318             #  given by GanetiApiError. '\\1' is a group substitution
 319             #  which places the first group '('|\")' in it's place.
 320             comp = re.compile("('|\")(?P<msg>.*)\\1")
 321             err = comp.search(str(e))
 322             # Any search that has 0 results will just return None.
 323             #   That is why we must check for err before proceeding.
 324             if err:
 325                 msg = err.groupdict()['msg']
 326                 self.error = msg
 327             else:
 328                 msg = str(e)
 329                 self.error = str(e)
 330             GanetiError.store_error(msg, obj=self, code=e.code)
 331
 332         else:
 333             if self.error:
 334                 self.error = None
 335                 GanetiError.objects.clear_errors(obj=self)
 336
 337     def _refresh(self):
 338         """
 339         Fetch raw data from the Ganeti cluster.
 340
 341         This must be implemented by children of this class.
 342         """
 343
 344         raise NotImplementedError
 345
 346     def check_job_status(self):
 347         if not self.last_job_id:
 348             return {}
 349
 350         ct = ContentType.objects.get_for_model(self)
 351         qs = Job.objects.filter(content_type=ct, object_id=self.pk)
 352         jobs = qs.order_by("job_id")
 353
 354         updates = {}
 355         op = None
 356         status = 'unknown'
 357
 358         for job in jobs:
 359             try:
 360                 data = self.rapi.GetJobStatus(job.job_id)
 361
 362                 if Job.valid_job(data):
 363                     op = data['ops'][-1]['OP_ID']
 364                     status = data['status']
 365
 366             except GanetiApiError:
 367                 pass
 368
 369             if status in ('success', 'error'):
 370                 for k, v in Job.parse_persistent_info(data).items():
 371                     setattr(job, k, v)
 372
 373             if status == 'unknown':
 374                 job.status = "unknown"
 375                 job.ignore_cache = False
 376
 377             if status in ('success', 'error', 'unknown'):
 378                 _updates = self._complete_job(self.cluster_id,
 379                                               self.hostname, op, status)
 380                 # XXX if the delete flag is set in updates then delete this
 381                 # model this happens here because _complete_job cannot delete
 382                 # this model
 383                 if _updates:
 384                     if 'deleted' in _updates:
 385                         # Delete ourselves. Also delete the job that caused us
 386                         # to delete ourselves; see #8439 for "fun" details.
 387                         # Order matters; the job's deletion cascades over us.
 388                         # Revisit that when we finally nuke all this caching
 389                         # bullshit.
 390                         self.delete()
 391                         job.delete()
 392                     else:
 393                         updates.update(_updates)
 394
 395         # we only care about the very last job for resetting the cache flags
 396         if not jobs or status in ('success', 'error', 'unknown'):
 397             updates['ignore_cache'] = False
 398             updates['last_job'] = None
 399
 400         return updates
 401
 402     @classmethod
 403     def _complete_job(cls, cluster_id, hostname, op, status):
 404         """
 405         Process a completed job.  This method will make any updates to related
 406         classes (like deleting an instance template) and return any data that
 407         should be updated.  This is a class method so that this processing can
 408         be done without a full instance.
 409
 410         @returns dict of updated values
 411         """
 412
 413         pass
 414
 415     def parse_transient_info(self):
 416         """
 417         Parse properties from cached info that is stored on the class but not
 418         in the database.
 419
 420         These properties will be loaded every time the object is instantiated.
 421         Properties stored on the class cannot be search efficiently via the
 422         django query api.
 423
 424         This method is specific to the child object.
 425         """
 426
 427         info_ = self.info
 428         # XXX ganeti 2.1 ctime is always None
 429         # XXX this means that we could nuke the conditionals!
 430         if info_['ctime'] is not None:
 431             self.ctime = datetime.fromtimestamp(info_['ctime'])
 432
 433     @classmethod
 434     def parse_persistent_info(cls, info):
 435         """
 436         Parse properties from cached info that are stored in the database.
 437
 438         These properties will be searchable by the django query api.
 439
 440         This method is specific to the child object.
 441         """
 442
 443         # mtime is sometimes None if object has never been modified
 444         if info['mtime'] is None:
 445             return {'mtime': None}
 446         return {'mtime': datetime.fromtimestamp(info['mtime'])}
 447
 448
 449 class JobManager(models.Manager):
 450     """
 451     Custom manager for Ganeti Jobs model
 452     """
 453     def create(self, **kwargs):
 454         """ helper method for creating a job with disabled cache """
 455         job = Job(ignore_cache=True, **kwargs)
 456         job.save(force_insert=True)
 457         return job
 458
 459
 460 class Job(CachedClusterObject):
 461     """
 462     model representing a job being run on a ganeti Cluster.  This includes
 463     operations such as creating or delting a virtual machine.
 464
 465     Jobs are a special type of CachedClusterObject.  Job's run once then become
 466     immutable.  The lazy cache is modified to become permanent once a complete
 467     status (success/error) has been detected.  The cache can be disabled by
 468     settning ignore_cache=True.
 469     """
 470
 471     job_id = models.IntegerField()
 472     content_type = models.ForeignKey(ContentType, related_name="+")
 473     object_id = models.IntegerField()
 474     obj = GenericForeignKey('content_type', 'object_id')
 475     cluster = models.ForeignKey('Cluster', related_name='jobs', editable=False)
 476     cluster_hash = models.CharField(max_length=40, editable=False)
 477
 478     finished = models.DateTimeField(null=True, blank=True)
 479     status = models.CharField(max_length=10)
 480     op = models.CharField(max_length=50)
 481
 482     objects = JobManager()
 483
 484     def save(self, *args, **kwargs):
 485         """
 486         sets the cluster_hash for newly saved instances
 487         """
 488         if self.id is None or self.cluster_hash == '':
 489             self.cluster_hash = self.cluster.hash
 490
 491         super(Job, self).save(*args, **kwargs)
 492
 493     @models.permalink
 494     def get_absolute_url(self):
 495         job = '%s/job/(?P<job_id>\d+)' % self.cluster
 496
 497         return 'ganeti_web.views.jobs.detail', (), {'job': job}
 498
 499     @property
 500     def rapi(self):
 501         return get_rapi(self.cluster_hash, self.cluster_id)
 502
 503     def _refresh(self):
 504         return self.rapi.GetJobStatus(self.job_id)
 505
 506     def load_info(self):
 507         """
 508         Load info for class.  This will load from ganeti if ignore_cache==True,
 509         otherwise this will always load from the cache.
 510         """
 511         if self.id and (self.ignore_cache or self.info is None):
 512             try:
 513                 self.refresh()
 514             except GanetiApiError, e:
 515                 # if the Job has been archived then we don't know whether it
 516                 # was successful or not. Mark it as unknown.
 517                 if e.code == 404:
 518                     self.status = 'unknown'
 519                     self.save()
 520                 else:
 521                     # its possible the cluster or crednetials are bad. fail
 522                     # silently
 523                     pass
 524
 525     def refresh(self):
 526         info = self._refresh()
 527         valid = self.valid_job(info)
 528         if valid:
 529             self.info = info
 530             self.save()
 531         # else:
 532         #     Job.objects.get(job_id=self.info['id']).delete()
 533
 534     @classmethod
 535     def valid_job(cls, info):
 536         status = info.get('status')
 537         ops = info.get('ops')
 538         return not (ops is None and status is None)
 539
 540     @classmethod
 541     def parse_op(cls, info):
 542         ops = info['ops']
 543         op = None
 544         if ops:
 545             # Return the most recent operation
 546             op = ops[-1]['OP_ID']
 547         return op
 548
 549     @classmethod
 550     def parse_persistent_info(cls, info):
 551         """
 552         Parse status and turn off cache bypass flag if job has finished
 553         """
 554         if not cls.valid_job(info):
 555             return
 556         op = cls.parse_op(info)
 557         data = {'status': info['status'], 'op': op}
 558         if data['status'] in ('error', 'success'):
 559             data['ignore_cache'] = False
 560         if info['end_ts']:
 561             data['finished'] = cls.parse_end_timestamp(info)
 562         return data
 563
 564     @staticmethod
 565     def parse_end_timestamp(info):
 566         sec, micro = info['end_ts']
 567         return datetime.fromtimestamp(sec + (micro / 1000000.0))
 568
 569     def parse_transient_info(self):
 570         pass
 571
 572     @property
 573     def current_operation(self):
 574         """
 575         Jobs may consist of multiple commands/operations.  This helper
 576         method will return the operation that is currently running or errored
 577         out, or the last operation if all operations have completed
 578
 579         @returns raw name of the current operation
 580         """
 581         info = self.info
 582         index = 0
 583         for i in range(len(info['opstatus'])):
 584             if info['opstatus'][i] != 'success':
 585                 index = i
 586                 break
 587         return info['ops'][index]['OP_ID']
 588
 589     @property
 590     def operation(self):
 591         """
 592         Returns the last operation, which is generally the primary operation.
 593         """
 594         return self.parse_op(self.info)
 595
 596     def __repr__(self):
 597         return "<Job %d (%d), status %r>" % (self.id, self.job_id,
 598                                              self.status)
 599
 600     __unicode__ = __repr__
 601
 602
 603 class VirtualMachine(CachedClusterObject):
 604     """
 605     The VirtualMachine (VM) model represents VMs within a Ganeti cluster.
 606
 607     The majority of properties are a cache for data stored in the cluster.
 608     All data retrieved via the RAPI is stored in VirtualMachine.info, and
 609     serialized automatically into VirtualMachine.serialized_info.
 610
 611     Attributes that need to be searchable should be stored as model fields.
 612     All other attributes will be stored within VirtualMachine.info.
 613
 614     This object uses a lazy update mechanism on instantiation.  If the cached
 615     info from the Ganeti cluster has expired, it will trigger an update.  This
 616     allows the cache to function in the absence of a periodic update mechanism
 617     such as Cron, Celery, or Threads.
 618
 619     XXX Serialized_info can possibly be changed to a CharField if an upper
 620         limit can be determined. (Later Date, if it will optimize db)
 621
 622     """
 623     cluster = models.ForeignKey('Cluster', related_name='virtual_machines',
 624                                 editable=False, default=0)
 625     hostname = models.CharField(max_length=128, db_index=True)
 626     owner = models.ForeignKey('ClusterUser', related_name='virtual_machines',
 627                               null=True, blank=True,
 628                               on_delete=models.SET_NULL)
 629     virtual_cpus = models.IntegerField(default=-1)
 630     disk_size = models.IntegerField(default=-1)
 631     ram = models.IntegerField(default=-1)
 632     minram = models.IntegerField(default=-1)
 633     cluster_hash = models.CharField(max_length=40, editable=False)
 634     operating_system = models.CharField(max_length=128)
 635     status = models.CharField(max_length=14)
 636
 637     # node relations
 638     primary_node = models.ForeignKey('Node', related_name='primary_vms',
 639                                      null=True, blank=True)
 640     secondary_node = models.ForeignKey('Node', related_name='secondary_vms',
 641                                        null=True, blank=True)
 642
 643     # The last job reference indicates that there is at least one pending job
 644     # for this virtual machine.  There may be more than one job, and that can
 645     # never be prevented.  This just indicates that job(s) are pending and the
 646     # job related code should be run (status, cleanup, etc).
 647     last_job = models.ForeignKey('Job', related_name="+", null=True,
 648                                  blank=True)
 649
 650     # deleted flag indicates a VM is being deleted, but the job has not
 651     # completed yet.  VMs that have pending_delete are still displayed in lists
 652     # and counted in quotas, but only so status can be checked.
 653     pending_delete = models.BooleanField(default=False)
 654     deleted = False
 655
 656     # Template temporarily stores parameters used to create this virtual
 657     # machine. This template is used to recreate the values entered into the
 658     # form.
 659     template = models.ForeignKey("VirtualMachineTemplate",
 660                                  related_name="instances", null=True,
 661                                  blank=True)
 662
 663     class Meta:
 664         ordering = ["hostname"]
 665         unique_together = (("cluster", "hostname"),)
 666
 667     def __unicode__(self):
 668         return self.hostname
 669
 670     def save(self, *args, **kwargs):
 671         """
 672         sets the cluster_hash for newly saved instances
 673         """
 674         if self.id is None:
 675             self.cluster_hash = self.cluster.hash
 676
 677         info_ = self.info
 678         if info_:
 679             found = False
 680             remove = []
 681             if self.cluster.username:
 682                 for tag in info_['tags']:
 683                     # Update owner Tag. Make sure the tag is set to the owner
 684                     #  that is set in webmgr.
 685                     if tag.startswith(constants.OWNER_TAG):
 686                         id = int(tag[len(constants.OWNER_TAG):])
 687                         # Since there is no 'update tag' delete old tag and
 688                         #  replace with tag containing correct owner id.
 689                         if id == self.owner_id:
 690                             found = True
 691                         else:
 692                             remove.append(tag)
 693                 if remove:
 694                     self.rapi.DeleteInstanceTags(self.hostname, remove)
 695                     for tag in remove:
 696                         info_['tags'].remove(tag)
 697                 if self.owner_id and not found:
 698                     tag = '%s%s' % (constants.OWNER_TAG, self.owner_id)
 699                     self.rapi.AddInstanceTags(self.hostname, [tag])
 700                     self.info['tags'].append(tag)
 701
 702         super(VirtualMachine, self).save(*args, **kwargs)
 703
 704     @models.permalink
 705     def get_absolute_url(self):
 706         """
 707         Return absolute url for this instance.
 708         """
 709
 710         return 'instance-detail', (), {'cluster_slug': self.cluster.slug,
 711                                        'instance': self.hostname}
 712
 713     @property
 714     def rapi(self):
 715         return get_rapi(self.cluster_hash, self.cluster_id)
 716
 717     @property
 718     def is_running(self):
 719         return self.status == 'running'
 720
 721     @classmethod
 722     def parse_persistent_info(cls, info):
 723         """
 724         Loads all values from cached info, included persistent properties that
 725         are stored in the database
 726         """
 727         data = super(VirtualMachine, cls).parse_persistent_info(info)
 728
 729         # Parse resource properties
 730         data['ram'] = info['beparams']['memory']
 731         data['virtual_cpus'] = info['beparams']['vcpus']
 732         # Sum up the size of each disk used by the VM
 733         disk_size = 0
 734         for disk in info['disk.sizes']:
 735             disk_size += disk
 736         data['disk_size'] = disk_size
 737         data['operating_system'] = info['os']
 738         data['status'] = info['status']
 739
 740         primary = info['pnode']
 741         if primary:
 742             try:
 743                 data['primary_node'] = Node.objects.get(hostname=primary)
 744             except Node.DoesNotExist:
 745                 # node is not created yet.  fail silently
 746                 data['primary_node'] = None
 747         else:
 748             data['primary_node'] = None
 749
 750         secondary = info['snodes']
 751         if len(secondary):
 752             secondary = secondary[0]
 753             try:
 754                 data['secondary_node'] = Node.objects.get(hostname=secondary)
 755             except Node.DoesNotExist:
 756                 # node is not created yet.  fail silently
 757                 data['secondary_node'] = None
 758         else:
 759             data['secondary_node'] = None
 760
 761         return data
 762
 763     @classmethod
 764     def _complete_job(cls, cluster_id, hostname, op, status):
 765         """
 766         if the cache bypass is enabled then check the status of the last job
 767         when the job is complete we can reenable the cache.
 768
 769         @returns - dictionary of values that were updates
 770         """
 771
 772         if status == 'unknown':
 773             # unknown status, the job was archived before it's final status
 774             # was polled.  Impossible to tell what happened.  Clear the job
 775             # so it is no longer polled.
 776             #
 777             # XXX This VM might be added by the CLI and be in an invalid
 778             # pending_delete state.  clearing pending_delete prevents this
 779             # but will result in "missing" vms in some cases.
 780             return dict(pending_delete=False)
 781
 782         base = VirtualMachine.objects.filter(cluster=cluster_id,
 783                                              hostname=hostname)
 784         if op == 'OP_INSTANCE_REMOVE':
 785             if status == 'success':
 786                 # XXX can't actually delete here since it would cause a
 787                 # recursive loop
 788                 return dict(deleted=True)
 789
 790         elif op == 'OP_INSTANCE_CREATE' and status == 'success':
 791             # XXX must update before deleting the template to maintain
 792             # referential integrity.  as a consequence return no other
 793             # updates.
 794             base.update(template=None)
 795             VirtualMachineTemplate.objects \
 796                 .filter(instances__hostname=hostname,
 797                         instances__cluster=cluster_id) \
 798                 .delete()
 799             return dict(template=None)
 800         return
 801
 802     def _refresh(self):
 803         # XXX if delete is pending then no need to refresh this object.
 804         if self.pending_delete or self.template_id:
 805             return None
 806         return self.rapi.GetInstance(self.hostname)
 807
 808     def shutdown(self, timeout=None):
 809         if timeout is None:
 810             id = self.rapi.ShutdownInstance(self.hostname)
 811         else:
 812             id = self.rapi.ShutdownInstance(self.hostname, timeout=timeout)
 813
 814         job = Job.objects.create(job_id=id, obj=self,
 815                                  cluster_id=self.cluster_id)
 816         self.last_job = job
 817         VirtualMachine.objects.filter(pk=self.id) \
 818             .update(last_job=job, ignore_cache=True)
 819         return job
 820
 821     def startup(self):
 822         id = self.rapi.StartupInstance(self.hostname)
 823         job = Job.objects.create(job_id=id, obj=self,
 824                                  cluster_id=self.cluster_id)
 825         self.last_job = job
 826         VirtualMachine.objects.filter(pk=self.id) \
 827             .update(last_job=job, ignore_cache=True)
 828         return job
 829
 830     def reboot(self):
 831         id = self.rapi.RebootInstance(self.hostname)
 832         job = Job.objects.create(job_id=id, obj=self,
 833                                  cluster_id=self.cluster_id)
 834         self.last_job = job
 835         VirtualMachine.objects.filter(pk=self.id) \
 836             .update(last_job=job, ignore_cache=True)
 837         return job
 838
 839     def migrate(self, mode='live', cleanup=False):
 840         """
 841         Migrates this VirtualMachine to another node.
 842
 843         Only works if the disk type is DRDB.
 844
 845         @param mode: live or non-live
 846         @param cleanup: clean up a previous migration, default is False
 847         """
 848         id = self.rapi.MigrateInstance(self.hostname, mode, cleanup)
 849         job = Job.objects.create(job_id=id, obj=self,
 850                                  cluster_id=self.cluster_id)
 851         self.last_job = job
 852         VirtualMachine.objects.filter(pk=self.id) \
 853             .update(last_job=job, ignore_cache=True)
 854         return job
 855
 856     def replace_disks(self, mode=REPLACE_DISK_AUTO, disks=None, node=None,
 857                       iallocator=None):
 858         id = self.rapi.ReplaceInstanceDisks(self.hostname, disks, mode, node,
 859                                             iallocator)
 860         job = Job.objects.create(job_id=id, obj=self,
 861                                  cluster_id=self.cluster_id)
 862         self.last_job = job
 863         VirtualMachine.objects.filter(pk=self.id) \
 864             .update(last_job=job, ignore_cache=True)
 865         return job
 866
 867     def setup_ssh_forwarding(self, sport=0):
 868         """
 869         Poke a proxy to start SSH forwarding.
 870
 871         Returns None if no proxy is configured, or if there was an error
 872         contacting the proxy.
 873         """
 874
 875         command = self.rapi.GetInstanceConsole(self.hostname)["command"]
 876
 877         if settings.VNC_PROXY:
 878             proxy_server = settings.VNC_PROXY.split(":")
 879             password = generate_random_password()
 880             sport = request_ssh(proxy_server, sport, self.info["pnode"],
 881                                 self.info["network_port"], password, command)
 882
 883             if sport:
 884                 return proxy_server[0], sport, password
 885
 886     def setup_vnc_forwarding(self, sport=0, tls=False):
 887         """
 888         Obtain VNC forwarding information, optionally configuring a proxy.
 889
 890         Returns None if a proxy is configured and there was an error
 891         contacting the proxy.
 892         """
 893
 894         password = ''
 895         info_ = self.info
 896         port = info_['network_port']
 897         node = info_['pnode']
 898
 899         # use proxy for VNC connection
 900         if settings.VNC_PROXY:
 901             proxy_server = settings.VNC_PROXY.split(":")
 902             password = generate_random_password()
 903             result = request_forwarding(proxy_server, node, port, password,
 904                                         sport=sport, tls=tls)
 905             if result:
 906                 return proxy_server[0], int(result), password
 907         else:
 908             return node, port, password
 909
 910     def __repr__(self):
 911         return "<VirtualMachine: '%s'>" % self.hostname
 912
 913
 914 class Node(CachedClusterObject):
 915     """
 916     The Node model represents nodes within a Ganeti cluster.
 917
 918     The majority of properties are a cache for data stored in the cluster.
 919     All data retrieved via the RAPI is stored in VirtualMachine.info, and
 920     serialized automatically into VirtualMachine.serialized_info.
 921
 922     Attributes that need to be searchable should be stored as model fields.
 923     All other attributes will be stored within VirtualMachine.info.
 924     """
 925
 926     ROLE_CHOICES = ((k, v) for k, v in constants.NODE_ROLE_MAP.items())
 927
 928     cluster = models.ForeignKey('Cluster', related_name='nodes')
 929     hostname = models.CharField(max_length=128, unique=True)
 930     cluster_hash = models.CharField(max_length=40, editable=False)
 931     offline = models.BooleanField()
 932     role = models.CharField(max_length=1, choices=ROLE_CHOICES)
 933     ram_total = models.IntegerField(default=-1)
 934     ram_free = models.IntegerField(default=-1)
 935     disk_total = models.IntegerField(default=-1)
 936     disk_free = models.IntegerField(default=-1)
 937     cpus = models.IntegerField(null=True, blank=True)
 938
 939     # The last job reference indicates that there is at least one pending job
 940     # for this virtual machine.  There may be more than one job, and that can
 941     # never be prevented.  This just indicates that job(s) are pending and the
 942     # job related code should be run (status, cleanup, etc).
 943     last_job = models.ForeignKey('Job', related_name="+", null=True,
 944                                  blank=True)
 945
 946     def __unicode__(self):
 947         return self.hostname
 948
 949     def save(self, *args, **kwargs):
 950         """
 951         sets the cluster_hash for newly saved instances
 952         """
 953         if self.id is None:
 954             self.cluster_hash = self.cluster.hash
 955         super(Node, self).save(*args, **kwargs)
 956
 957     @models.permalink
 958     def get_absolute_url(self):
 959         """
 960         Return absolute url for this node.
 961         """
 962
 963         return 'node-detail', (), {'cluster_slug': self.cluster.slug,
 964                                    'host': self.hostname}
 965
 966     def _refresh(self):
 967         """ returns node info from the ganeti server """
 968         return self.rapi.GetNode(self.hostname)
 969
 970     @property
 971     def rapi(self):
 972         return get_rapi(self.cluster_hash, self.cluster_id)
 973
 974     @classmethod
 975     def parse_persistent_info(cls, info):
 976         """
 977         Loads all values from cached info, included persistent properties that
 978         are stored in the database
 979         """
 980         data = super(Node, cls).parse_persistent_info(info)
 981
 982         # Parse resource properties
 983         data['ram_total'] = info.get("mtotal") or 0
 984         data['ram_free'] = info.get("mfree") or 0
 985         data['disk_total'] = info.get("dtotal") or 0
 986         data['disk_free'] = info.get("dfree") or 0
 987         data['cpus'] = info.get("csockets")
 988         data['offline'] = info['offline']
 989         data['role'] = info['role']
 990         return data
 991
 992     @property
 993     def ram(self):
 994         """ returns dict of free and total ram """
 995         values = VirtualMachine.objects \
 996             .filter(Q(primary_node=self) | Q(secondary_node=self)) \
 997             .filter(status='running') \
 998             .exclude(ram=-1).order_by() \
 999             .aggregate(used=Sum('ram'))
1000
1001         total = self.ram_total
1002         used = total - self.ram_free
1003         allocated = values.get("used") or 0
1004         free = total - allocated if allocated >= 0 and total >= 0 else -1
1005
1006         return {
1007             'total': total,
1008             'free':  free,
1009             'allocated': allocated,
1010             'used': used,
1011         }
1012
1013     @property
1014     def disk(self):
1015         """ returns dict of free and total disk space """
1016         values = VirtualMachine.objects \
1017             .filter(Q(primary_node=self) | Q(secondary_node=self)) \
1018             .exclude(disk_size=-1).order_by() \
1019             .aggregate(used=Sum('disk_size'))
1020
1021         total = self.disk_total
1022         used = total - self.disk_free
1023         allocated = values.get("used") or 0
1024         free = total - allocated if allocated >= 0 and total >= 0 else -1
1025
1026         return {
1027             'total': total,
1028             'free':  free,
1029             'allocated': allocated,
1030             'used': used,
1031         }
1032
1033     @property
1034     def allocated_cpus(self):
1035         values = VirtualMachine.objects \
1036             .filter(primary_node=self, status='running') \
1037             .exclude(virtual_cpus=-1).order_by() \
1038             .aggregate(cpus=Sum('virtual_cpus'))
1039         return values.get("cpus") or 0
1040
1041     def set_role(self, role, force=False):
1042         """
1043         Sets the role for this node
1044
1045         @param role - one of the following choices:
1046             * master
1047             * master-candidate
1048             * regular
1049             * drained
1050             * offline
1051         """
1052         id = self.rapi.SetNodeRole(self.hostname, role, force)
1053         job = Job.objects.create(job_id=id, obj=self,
1054                                  cluster_id=self.cluster_id)
1055         self.last_job = job
1056         Node.objects.filter(pk=self.pk).update(ignore_cache=True, last_job=job)
1057         return job
1058
1059     def evacuate(self, iallocator=None, node=None):
1060         """
1061         migrates all secondary instances off this node
1062         """
1063         id = self.rapi.EvacuateNode(self.hostname, iallocator=iallocator,
1064                                     remote_node=node)
1065         job = Job.objects.create(job_id=id, obj=self,
1066                                  cluster_id=self.cluster_id)
1067         self.last_job = job
1068         Node.objects.filter(pk=self.pk) \
1069             .update(ignore_cache=True, last_job=job)
1070         return job
1071
1072     def migrate(self, mode=None):
1073         """
1074         migrates all primary instances off this node
1075         """
1076         id = self.rapi.MigrateNode(self.hostname, mode)
1077         job = Job.objects.create(job_id=id, obj=self,
1078                                  cluster_id=self.cluster_id)
1079         self.last_job = job
1080         Node.objects.filter(pk=self.pk).update(ignore_cache=True, last_job=job)
1081         return job
1082
1083     def __repr__(self):
1084         return "<Node: '%s'>" % self.hostname
1085
1086
1087 class Cluster(CachedClusterObject):
1088     """
1089     A Ganeti cluster that is being tracked by this manager tool
1090     """
1091     hostname = models.CharField(_('hostname'), max_length=128, unique=True)
1092     slug = models.SlugField(_('slug'), max_length=50, unique=True,
1093                             db_index=True)
1094     port = models.PositiveIntegerField(_('port'), default=5080)
1095     description = models.CharField(_('description'), max_length=128,
1096                                    blank=True)
1097     username = models.CharField(_('username'), max_length=128, blank=True)
1098     password = PatchedEncryptedCharField(_('password'), default="",
1099                                          max_length=128, blank=True)
1100     hash = models.CharField(_('hash'), max_length=40, editable=False)
1101
1102     # quota properties
1103     virtual_cpus = models.IntegerField(_('Virtual CPUs'), null=True,
1104                                        blank=True)
1105     disk = models.IntegerField(_('disk'), null=True, blank=True)
1106     ram = models.IntegerField(_('ram'), null=True, blank=True)
1107
1108     # The last job reference indicates that there is at least one pending job
1109     # for this virtual machine.  There may be more than one job, and that can
1110     # never be prevented.  This just indicates that job(s) are pending and the
1111     # job related code should be run (status, cleanup, etc).
1112     last_job = models.ForeignKey('Job', related_name='cluster_last_job',
1113                                  null=True, blank=True)
1114
1115     class Meta:
1116         ordering = ["hostname", "description"]
1117
1118     def __unicode__(self):
1119         return self.hostname
1120
1121     def save(self, *args, **kwargs):
1122         self.hash = self.create_hash()
1123         super(Cluster, self).save(*args, **kwargs)
1124
1125     @models.permalink
1126     def get_absolute_url(self):
1127         return 'cluster-detail', (), {'cluster_slug': self.slug}
1128
1129     # XXX probably hax
1130     @property
1131     def cluster_id(self):
1132         return self.id
1133
1134     @classmethod
1135     def decrypt_password(cls, value):
1136         """
1137         Convenience method for decrypting a password without an instance.
1138         This was partly cribbed from django-fields which only allows decrypting
1139         from a model instance.
1140
1141         If the password appears to be encrypted, this method will decrypt it;
1142         otherwise, it will return the password unchanged.
1143
1144         This method is bonghits.
1145         """
1146
1147         field, chaff, chaff, chaff = cls._meta.get_field_by_name('password')
1148
1149         if value.startswith(field.prefix):
1150             ciphertext = value[len(field.prefix):]
1151             plaintext = field.cipher.decrypt(binascii.a2b_hex(ciphertext))
1152             password = plaintext.split('\0')[0]
1153         else:
1154             password = value
1155
1156         return force_unicode(password)
1157
1158     @property
1159     def rapi(self):
1160         """
1161         retrieves the rapi client for this cluster.
1162         """
1163         # XXX always pass self in.  not only does it avoid querying this object
1164         # from the DB a second time, it also prevents a recursion loop caused
1165         # by __init__ fetching info from the Cluster
1166         return get_rapi(self.hash, self)
1167
1168     def create_hash(self):
1169         """
1170         Creates a hash for this cluster based on credentials required for
1171         connecting to the server
1172         """
1173         s = '%s%s%s%s' % (self.username, self.password, self.hostname,
1174                           self.port)
1175         return sha1(s).hexdigest()
1176
1177     def get_default_quota(self):
1178         """
1179         Returns the default quota for this cluster
1180         """
1181         return {
1182             "default": 1,
1183             "ram": self.ram,
1184             "disk": self.disk,
1185             "virtual_cpus": self.virtual_cpus,
1186         }
1187
1188     def get_quota(self, user=None):
1189         """
1190         Get the quota for a ClusterUser
1191
1192         @return user's quota, default quota, or none
1193         """
1194         if user is None:
1195             return self.get_default_quota()
1196
1197         # attempt to query user specific quota first. if it does not exist
1198         # then fall back to the default quota
1199         query = Quota.objects.filter(cluster=self, user=user)
1200         quotas = query.values('ram', 'disk', 'virtual_cpus')
1201         if quotas:
1202             quota = quotas[0]
1203             quota['default'] = 0
1204             return quota
1205
1206         return self.get_default_quota()
1207
1208     def set_quota(self, user, data):
1209         """
1210         Set the quota for a ClusterUser.
1211
1212         If data is None, the quota will be removed.
1213
1214         @param values: dictionary of values, or None to delete the quota
1215         """
1216
1217         kwargs = {'cluster': self, 'user': user}
1218         if data is None:
1219             Quota.objects.filter(**kwargs).delete()
1220         else:
1221             quota, new = Quota.objects.get_or_create(**kwargs)
1222             quota.__dict__.update(data)
1223             quota.save()
1224
1225     @classmethod
1226     def get_quotas(cls, clusters=None, user=None):
1227         """ retrieve a bulk list of cluster quotas """
1228
1229         if clusters is None:
1230             clusters = Cluster.objects.all()
1231
1232         quotas = {}
1233         cluster_id_map = {}
1234         for cluster in clusters:
1235             quotas[cluster] = {
1236                 'default': 1,
1237                 'ram': cluster.ram,
1238                 'disk': cluster.disk,
1239                 'virtual_cpus': cluster.virtual_cpus,
1240             }
1241             cluster_id_map[cluster.id] = cluster
1242
1243         # get user's custom queries if any
1244         if user is not None:
1245             qs = Quota.objects.filter(cluster__in=clusters, user=user)
1246             values = qs.values('ram', 'disk', 'virtual_cpus', 'cluster__id')
1247
1248             for custom in values:
1249                 try:
1250                     cluster = cluster_id_map[custom['cluster__id']]
1251                 except KeyError:
1252                     continue
1253                 custom['default'] = 0
1254                 del custom['cluster__id']
1255                 quotas[cluster] = custom
1256
1257         return quotas
1258
1259     def sync_virtual_machines(self, remove=False):
1260         """
1261         Synchronizes the VirtualMachines in the database with the information
1262         this ganeti cluster has:
1263             * VMs no longer in ganeti are deleted
1264             * VMs missing from the database are added
1265         """
1266         ganeti = self.instances()
1267         db = self.virtual_machines.all().values_list('hostname', flat=True)
1268
1269         # add VMs missing from the database
1270         for hostname in filter(lambda x: unicode(x) not in db, ganeti):
1271             vm = VirtualMachine.objects.create(cluster=self, hostname=hostname)
1272             vm.refresh()
1273
1274         # deletes VMs that are no longer in ganeti
1275         if remove:
1276             missing_ganeti = filter(lambda x: str(x) not in ganeti, db)
1277             if missing_ganeti:
1278                 self.virtual_machines \
1279                     .filter(hostname__in=missing_ganeti).delete()
1280
1281     def sync_nodes(self, remove=False):
1282         """
1283         Synchronizes the Nodes in the database with the information
1284         this ganeti cluster has:
1285             * Nodes no longer in ganeti are deleted
1286             * Nodes missing from the database are added
1287         """
1288         ganeti = self.rapi.GetNodes()
1289         db = self.nodes.all().values_list('hostname', flat=True)
1290
1291         # add Nodes missing from the database
1292         for hostname in filter(lambda x: unicode(x) not in db, ganeti):
1293             node = Node.objects.create(cluster=self, hostname=hostname)
1294             node.refresh()
1295
1296         # deletes Nodes that are no longer in ganeti
1297         if remove:
1298             missing_ganeti = filter(lambda x: str(x) not in ganeti, db)
1299             if missing_ganeti:
1300                 self.nodes.filter(hostname__in=missing_ganeti).delete()
1301
1302     @property
1303     def missing_in_ganeti(self):
1304         """
1305         Returns a list of VirtualMachines that are missing from the Ganeti
1306         cluster but present in the database.
1307         """
1308         ganeti = self.instances()
1309         qs = self.virtual_machines.exclude(template__isnull=False)
1310         db = qs.values_list('hostname', flat=True)
1311         return [x for x in db if str(x) not in ganeti]
1312
1313     @property
1314     def missing_in_db(self):
1315         """
1316         Returns list of VirtualMachines that are missing from the database, but
1317         present in ganeti
1318         """
1319         ganeti = self.instances()
1320         db = self.virtual_machines.all().values_list('hostname', flat=True)
1321         return [x for x in ganeti if unicode(x) not in db]
1322
1323     @property
1324     def nodes_missing_in_db(self):
1325         """
1326         Returns list of Nodes that are missing from the database, but present
1327         in ganeti.
1328         """
1329         try:
1330             ganeti = self.rapi.GetNodes()
1331         except GanetiApiError:
1332             ganeti = []
1333         db = self.nodes.all().values_list('hostname', flat=True)
1334         return [x for x in ganeti if unicode(x) not in db]
1335
1336     @property
1337     def nodes_missing_in_ganeti(self):
1338         """
1339         Returns list of Nodes that are missing from the ganeti cluster
1340         but present in the database
1341         """
1342         try:
1343             ganeti = self.rapi.GetNodes()
1344         except GanetiApiError:
1345             ganeti = []
1346         db = self.nodes.all().values_list('hostname', flat=True)
1347         return filter(lambda x: str(x) not in ganeti, db)
1348
1349     @property
1350     def available_ram(self):
1351         """ returns dict of free and total ram """
1352         nodes = self.nodes.exclude(ram_total=-1) \
1353             .aggregate(total=Sum('ram_total'), free=Sum('ram_free'))
1354         total = max(nodes.get("total", 0), 0)
1355         free = max(nodes.get("free", 0), 0)
1356         used = total - free
1357         values = self.virtual_machines \
1358             .filter(status='running') \
1359             .exclude(ram=-1).order_by() \
1360             .aggregate(used=Sum('ram'))
1361
1362         if values.get("used") is None:
1363             allocated = 0
1364         else:
1365             allocated = values["used"]
1366
1367         free = max(total - allocated, 0)
1368
1369         return {
1370             'total': total,
1371             'free': free,
1372             'allocated': allocated,
1373             'used': used,
1374         }
1375
1376     @property
1377     def available_disk(self):
1378         """ returns dict of free and total disk space """
1379         nodes = self.nodes.exclude(disk_total=-1) \
1380             .aggregate(total=Sum('disk_total'), free=Sum('disk_free'))
1381         total = max(nodes.get("total", 0), 0)
1382         free = max(nodes.get("free", 0), 0)
1383         used = total - free
1384         values = self.virtual_machines \
1385             .exclude(disk_size=-1).order_by() \
1386             .aggregate(used=Sum('disk_size'))
1387
1388         if values.get("used") is None:
1389             allocated = 0
1390         else:
1391             allocated = values["used"]
1392
1393         free = max(total - allocated, 0)
1394
1395         return {
1396             'total': total,
1397             'free': free,
1398             'allocated': allocated,
1399             'used': used,
1400         }
1401
1402     def _refresh(self):
1403         return self.rapi.GetInfo()
1404
1405     def instances(self, bulk=False):
1406         """Gets all VMs which reside under the Cluster
1407         Calls the rapi client for all instances.
1408         """
1409         try:
1410             return self.rapi.GetInstances(bulk=bulk)
1411         except GanetiApiError:
1412             return []
1413
1414     def instance(self, instance):
1415         """Get a single Instance
1416         Calls the rapi client for a specific instance.
1417         """
1418         try:
1419             return self.rapi.GetInstance(instance)
1420         except GanetiApiError:
1421             return None
1422
1423     def redistribute_config(self):
1424         """
1425         Redistribute config from cluster's master node to all
1426         other nodes.
1427         """
1428         # no exception handling, because it's being done in a view
1429         id = self.rapi.RedistributeConfig()
1430         job = Job.objects.create(job_id=id, obj=self, cluster_id=self.id)
1431         self.last_job = job
1432         Cluster.objects.filter(pk=self.id) \
1433             .update(last_job=job, ignore_cache=True)
1434         return job
1435
1436
1437 class VirtualMachineTemplate(models.Model):
1438     """
1439     Virtual Machine Template holds all the values for the create virtual
1440     machine form so that they can automatically be used or edited by a user.
1441     """
1442
1443     template_name = models.CharField(max_length=255, default="")
1444     temporary = BooleanField(verbose_name=_("Temporary"), default=False)
1445     description = models.CharField(max_length=255, default="")
1446     cluster = models.ForeignKey(Cluster, related_name="templates", null=True,
1447                                 blank=True)
1448     start = models.BooleanField(verbose_name=_('Start up After Creation'),
1449                                 default=True)
1450     no_install = models.BooleanField(verbose_name=_('Do not install OS'),
1451                                      default=False)
1452     ip_check = BooleanField(verbose_name=_("IP Check"), default=True)
1453     name_check = models.BooleanField(verbose_name=_('DNS Name Check'),
1454                                      default=True)
1455     iallocator = models.BooleanField(verbose_name=_('Automatic Allocation'),
1456                                      default=False)
1457     iallocator_hostname = models.CharField(max_length=255, blank=True)
1458     disk_template = models.CharField(verbose_name=_('Disk Template'),
1459                                      max_length=16)
1460     # XXX why aren't these FKs?
1461     pnode = models.CharField(verbose_name=_('Primary Node'), max_length=255,
1462                              default="")
1463     snode = models.CharField(verbose_name=_('Secondary Node'), max_length=255,
1464                              default="")
1465     os = models.CharField(verbose_name=_('Operating System'), max_length=255)
1466
1467     # Backend parameters (BEPARAMS)
1468     vcpus = models.IntegerField(verbose_name=_('Virtual CPUs'),
1469                                 validators=[MinValueValidator(1)], null=True,
1470                                 blank=True)
1471     # XXX do we really want the minimum memory to be 100MiB? This isn't
1472     # strictly necessary AFAICT.
1473     memory = models.IntegerField(verbose_name=_('Memory'),
1474                                  validators=[MinValueValidator(100)],
1475                                  null=True, blank=True)
1476     minmem = models.IntegerField(verbose_name=_('Minimum Memory'),
1477                                  validators=[MinValueValidator(100)],
1478                                  null=True, blank=True)
1479     disks = PickleField(verbose_name=_('Disks'), null=True, blank=True)
1480     # XXX why isn't this an enum?
1481     disk_type = models.CharField(verbose_name=_('Disk Type'), max_length=255,
1482                                  default="")
1483     nics = PickleField(verbose_name=_('NICs'), null=True, blank=True)
1484     # XXX why isn't this an enum?
1485     nic_type = models.CharField(verbose_name=_('NIC Type'), max_length=255,
1486                                 default="")
1487
1488     # Hypervisor parameters (HVPARAMS)
1489     kernel_path = models.CharField(verbose_name=_('Kernel Path'),
1490                                    max_length=255, default="", blank=True)
1491     root_path = models.CharField(verbose_name=_('Root Path'), max_length=255,
1492                                  default='/', blank=True)
1493     serial_console = models.BooleanField(
1494         verbose_name=_('Enable Serial Console'))
1495     boot_order = models.CharField(verbose_name=_('Boot Device'),
1496                                   max_length=255, default="")
1497     cdrom_image_path = models.CharField(verbose_name=_('CD-ROM Image Path'),
1498                                         max_length=512, blank=True)
1499     cdrom2_image_path = models.CharField(
1500         verbose_name=_('CD-ROM 2 Image Path'),
1501         max_length=512, blank=True)
1502
1503     class Meta:
1504         unique_together = (("cluster", "template_name"),)
1505
1506     def __unicode__(self):
1507         if self.temporary:
1508             return u'(temporary)'
1509         else:
1510             return self.template_name
1511
1512     def set_name(self, name):
1513         """
1514         Set this template's name.
1515
1516         If the name is blank, this template will become temporary and its name
1517         will be set to a unique timestamp.
1518         """
1519
1520         if name:
1521             self.template_name = name
1522         else:
1523             # The template is temporary and will be removed by the VM when the
1524             # VM successfully comes into existence.
1525             self.temporary = True
1526             # Give it a temporary name. Something unique. This is the number
1527             # of microseconds since the epoch; I figure that it'll work out
1528             # alright.
1529             self.template_name = str(int(time.time() * (10 ** 6)))
1530
1531
1532 class GanetiError(models.Model):
1533     """
1534     Class for storing errors which occured in Ganeti
1535     """
1536     cluster = models.ForeignKey(Cluster, related_name="errors")
1537     msg = models.TextField()
1538     code = models.PositiveIntegerField(blank=True, null=True)
1539
1540     # XXX could be fixed with django-model-util's TimeStampedModel
1541     timestamp = models.DateTimeField()
1542
1543     # determines if the errors still appears or not
1544     cleared = models.BooleanField(default=False)
1545
1546     # cluster object (cluster, VM, Node) affected by the error (if any)
1547     obj_type = models.ForeignKey(ContentType, related_name="ganeti_errors")
1548     obj_id = models.PositiveIntegerField()
1549     obj = GenericForeignKey("obj_type", "obj_id")
1550
1551     objects = QuerySetManager()
1552
1553     class Meta:
1554         ordering = ("-timestamp", "code", "msg")
1555
1556     def __unicode__(self):
1557         base = u"[%s] %s" % (self.timestamp, self.msg)
1558         return base
1559
1560     class QuerySet(QuerySet):
1561
1562         def clear_errors(self, obj=None):
1563             """
1564             Clear errors instead of deleting them.
1565             """
1566
1567             qs = self.filter(cleared=False)
1568
1569             if obj:
1570                 qs = qs.get_errors(obj)
1571
1572             return qs.update(cleared=True)
1573
1574         def get_errors(self, obj):
1575             """
1576             Manager method used for getting QuerySet of all errors depending
1577             on passed arguments.
1578
1579             @param  obj   affected object (itself or just QuerySet)
1580             """
1581
1582             if obj is None:
1583                 raise RuntimeError("Implementation error calling get_errors()"
1584                                    "with None")
1585
1586             # Create base query of errors to return.
1587             #
1588             # if it's a Cluster or a queryset for Clusters, then we need to
1589             # get all errors from the Clusters. Do this by filtering on
1590             # GanetiError.cluster instead of obj_id.
1591             if isinstance(obj, (Cluster,)):
1592                 return self.filter(cluster=obj)
1593
1594             elif isinstance(obj, (QuerySet,)):
1595                 if obj.model == Cluster:
1596                     return self.filter(cluster__in=obj)
1597                 else:
1598                     ct = ContentType.objects.get_for_model(obj.model)
1599                     return self.filter(obj_type=ct, obj_id__in=obj)
1600
1601             else:
1602                 ct = ContentType.objects.get_for_model(obj.__class__)
1603                 return self.filter(obj_type=ct, obj_id=obj.pk)
1604
1605     def __repr__(self):
1606         return "<GanetiError '%s'>" % self.msg
1607
1608     @classmethod
1609     def store_error(cls, msg, obj, code, **kwargs):
1610         """
1611         Create and save an error with the given information.
1612
1613         @param  msg  error's message
1614         @param  obj  object (i.e. cluster or vm) affected by the error
1615         @param code  error's code number
1616         """
1617         ct = ContentType.objects.get_for_model(obj.__class__)
1618         is_cluster = isinstance(obj, Cluster)
1619
1620         # 401 -- bad permissions
1621         # 401 is cluster-specific error and thus shouldn't appear on any other
1622         # object.
1623         if code == 401:
1624             if not is_cluster:
1625                 # NOTE: what we do here is almost like:
1626                 #  return self.store_error(msg=msg, code=code, obj=obj.cluster)
1627                 # we just omit the recursiveness
1628                 obj = obj.cluster
1629                 ct = ContentType.objects.get_for_model(Cluster)
1630                 is_cluster = True
1631
1632         # 404 -- object not found
1633         # 404 can occur on any object, but when it occurs on a cluster, then
1634         # any of its children must not see the error again
1635         elif code == 404:
1636             if not is_cluster:
1637                 # return if the error exists for cluster
1638                 try:
1639                     c_ct = ContentType.objects.get_for_model(Cluster)
1640                     return cls.objects.filter(msg=msg, obj_type=c_ct,
1641                                               code=code,
1642                                               obj_id=obj.cluster_id,
1643                                               cleared=False)[0]
1644
1645                 except (cls.DoesNotExist, IndexError):
1646                     # we want to proceed when the error is not
1647                     # cluster-specific
1648                     pass
1649
1650         # XXX use a try/except instead of get_or_create().  get_or_create()
1651         # does not allow us to set cluster_id.  This means we'd have to query
1652         # the cluster object to create the error.  we can't guaranteee the
1653         # cluster will already be queried so use create() instead which does
1654         # allow cluster_id
1655         try:
1656             return cls.objects.filter(msg=msg, obj_type=ct, obj_id=obj.pk,
1657                                       code=code, **kwargs)[0]
1658
1659         except (cls.DoesNotExist, IndexError):
1660             cluster_id = obj.pk if is_cluster else obj.cluster_id
1661
1662             return cls.objects.create(timestamp=datetime.now(), msg=msg,
1663                                       obj_type=ct, obj_id=obj.pk,
1664                                       cluster_id=cluster_id, code=code,
1665                                       **kwargs)
1666
1667
1668 class ClusterUser(models.Model):
1669     """
1670     Base class for objects that may interact with a Cluster or VirtualMachine.
1671     """
1672
1673     name = models.CharField(max_length=128)
1674     real_type = models.ForeignKey(ContentType, related_name="+",
1675                                   editable=False, null=True, blank=True)
1676
1677     def __unicode__(self):
1678         return self.name
1679
1680     def save(self, *args, **kwargs):
1681         if not self.id:
1682             self.real_type = self._get_real_type()
1683         super(ClusterUser, self).save(*args, **kwargs)
1684
1685     def get_absolute_url(self):
1686         return self.cast().get_absolute_url()
1687
1688     @property
1689     def permissable(self):
1690         """ returns an object that can be granted permissions """
1691         return self.cast().permissable
1692
1693     @classmethod
1694     def _get_real_type(cls):
1695         return ContentType.objects.get_for_model(cls)
1696
1697     def cast(self):
1698         return self.real_type.get_object_for_this_type(pk=self.pk)
1699
1700     def used_resources(self, cluster=None, only_running=True):
1701         """
1702         Return dictionary of total resources used by VMs that this ClusterUser
1703         has perms to.
1704         @param cluster  if set, get only VMs from specified cluster
1705         @param only_running  if set, get only running VMs
1706         """
1707         # XXX - order_by must be cleared or it breaks annotation grouping since
1708         #       the default order_by field is also added to the group_by clause
1709         base = self.virtual_machines.all().order_by()
1710
1711         # XXX - use a custom aggregate for ram and vcpu count when filtering by
1712         # running.  this allows us to execute a single query.
1713         #
1714         # XXX - quotes must be used in this order.  postgresql quirk
1715         if only_running:
1716             sum_ram = SumIf('ram', condition="status='running'")
1717             sum_vcpus = SumIf('virtual_cpus', condition="status='running'")
1718         else:
1719             sum_ram = Sum('ram')
1720             sum_vcpus = Sum('virtual_cpus')
1721
1722         base = base.exclude(ram=-1, disk_size=-1, virtual_cpus=-1)
1723
1724         if cluster:
1725             base = base.filter(cluster=cluster)
1726             result = base.aggregate(ram=sum_ram, disk=Sum('disk_size'),
1727                                     virtual_cpus=sum_vcpus)
1728
1729             # repack with zeros instead of Nones
1730             if result['disk'] is None:
1731                 result['disk'] = 0
1732             if result['ram'] is None:
1733                 result['ram'] = 0
1734             if result['virtual_cpus'] is None:
1735                 result['virtual_cpus'] = 0
1736             return result
1737
1738         else:
1739             base = base.values('cluster').annotate(uram=sum_ram,
1740                                                    udisk=Sum('disk_size'),
1741                                                    uvirtual_cpus=sum_vcpus)
1742
1743             # repack as dictionary
1744             result = {}
1745             for used in base:
1746                 # repack with zeros instead of Nones, change index names
1747                 used["ram"] = used.pop("uram") or 0
1748                 used["disk"] = used.pop("udisk") or 0
1749                 used["virtual_cpus"] = used.pop("uvirtual_cpus") or 0
1750                 result[used.pop('cluster')] = used
1751
1752             return result
1753
1754
1755 class Profile(ClusterUser):
1756     """
1757     Profile associated with a django.contrib.auth.User object.
1758     """
1759     user = models.OneToOneField(User)
1760
1761     def get_absolute_url(self):
1762         return self.user.get_absolute_url()
1763
1764     def grant(self, perm, obj):
1765         self.user.grant(perm, obj)
1766
1767     def set_perms(self, perms, obj):
1768         self.user.set_perms(perms, obj)
1769
1770     def get_objects_any_perms(self, *args, **kwargs):
1771         return self.user.get_objects_any_perms(*args, **kwargs)
1772
1773     def has_perm(self, *args, **kwargs):
1774         return self.user.has_perm(*args, **kwargs)
1775
1776     @property
1777     def permissable(self):
1778         """ returns an object that can be granted permissions """
1779         return self.user
1780
1781
1782 class Organization(ClusterUser):
1783     """
1784     An organization is used for grouping Users.
1785
1786     Organizations are matched with an instance of contrib.auth.models.Group.
1787     This model exists so that contrib.auth.models.Group have a 1:1 relation
1788     with a ClusterUser on which quotas and permissions can be assigned.
1789     """
1790
1791     group = models.OneToOneField(Group, related_name='organization')
1792
1793     def get_absolute_url(self):
1794         return self.group.get_absolute_url()
1795
1796     def grant(self, perm, object):
1797         self.group.grant(perm, object)
1798
1799     def set_perms(self, perms, object):
1800         self.group.set_perms(perms, object)
1801
1802     def get_objects_any_perms(self, *args, **kwargs):
1803         return self.group.get_objects_any_perms(*args, **kwargs)
1804
1805     def has_perm(self, *args, **kwargs):
1806         return self.group.has_perm(*args, **kwargs)
1807
1808     @property
1809     def permissable(self):
1810         """ returns an object that can be granted permissions """
1811         return self.group
1812
1813
1814 class Quota(models.Model):
1815     """
1816     A resource limit imposed on a ClusterUser for a given Cluster.  The
1817     attributes of this model represent maximum values the ClusterUser can
1818     consume.  The absence of a Quota indicates unlimited usage.
1819     """
1820     user = models.ForeignKey(ClusterUser, related_name='quotas')
1821     cluster = models.ForeignKey(Cluster, related_name='quotas')
1822
1823     ram = models.IntegerField(default=0, null=True, blank=True)
1824     disk = models.IntegerField(default=0, null=True, blank=True)
1825     virtual_cpus = models.IntegerField(default=0, null=True, blank=True)
1826
1827
1828 class SSHKey(models.Model):
1829     """
1830     Model representing user's SSH public key. Virtual machines rely on
1831     many ssh keys.
1832     """
1833     key = models.TextField(validators=[validate_sshkey])
1834     #filename = models.CharField(max_length=128) # saves key file's name
1835     user = models.ForeignKey(User, related_name='ssh_keys')
1836
1837
1838 def create_profile(sender, instance, **kwargs):
1839     """
1840     Create a profile object whenever a new user is created, also keeps the
1841     profile name synchronized with the username
1842     """
1843     try:
1844         profile, new = Profile.objects.get_or_create(user=instance)
1845         if profile.name != instance.username:
1846             profile.name = instance.username
1847             profile.save()
1848     except DatabaseError:
1849         # XXX - since we're using south to track migrations the Profile table
1850         # won't be available the first time syncdb is run.  Catch the error
1851         # here and let the south migration handle it.
1852         pass
1853
1854
1855 def update_cluster_hash(sender, instance, **kwargs):
1856     """
1857     Updates the Cluster hash for all of it's VirtualMachines, Nodes, and Jobs
1858     """
1859     instance.virtual_machines.all().update(cluster_hash=instance.hash)
1860     instance.jobs.all().update(cluster_hash=instance.hash)
1861     instance.nodes.all().update(cluster_hash=instance.hash)
1862
1863
1864 def update_organization(sender, instance, **kwargs):
1865     """
1866     Creates a Organizations whenever a contrib.auth.models.Group is created
1867     """
1868     org, new = Organization.objects.get_or_create(group=instance)
1869     org.name = instance.name
1870     org.save()
1871
1872 post_save.connect(create_profile, sender=User)
1873 post_save.connect(update_cluster_hash, sender=Cluster)
1874 post_save.connect(update_organization, sender=Group)
1875
1876 # Disconnect create_default_site from django.contrib.sites so that
1877 #  the useless table for sites is not created. This will be
1878 #  reconnected for other apps to use in update_sites_module.
1879 post_syncdb.disconnect(create_default_site, sender=sites_app)
1880 post_syncdb.connect(management.update_sites_module, sender=sites_app,
1881                     dispatch_uid="ganeti.management.update_sites_module")
1882
1883
1884 def regenerate_cu_children(sender, **kwargs):
1885     """
1886     Resets may destroy Profiles and/or Organizations. We need to regenerate
1887     them.
1888     """
1889
1890     # So. What are we actually doing here?
1891     # Whenever a User or Group is saved, the associated Profile or
1892     # Organization is also updated. This means that, if a Profile for a User
1893     # is absent, it will be created.
1894     # More importantly, *why* might a Profile be missing? Simple. Resets of
1895     # the ganeti app destroy them. This shouldn't happen in production, and
1896     # only occasionally in development, but it's good to explicitly handle
1897     # this particular case so that missing Profiles not resulting from a reset
1898     # are easier to diagnose.
1899     try:
1900         for user in User.objects.filter(profile__isnull=True):
1901             user.save()
1902         for group in Group.objects.filter(organization__isnull=True):
1903             group.save()
1904     except DatabaseError:
1905         # XXX - since we're using south to track migrations the Profile table
1906         # won't be available the first time syncdb is run.  Catch the error
1907         # here and let the south migration handle it.
1908         pass
1909
1910 post_syncdb.connect(regenerate_cu_children)
1911
1912
1913 def log_group_create(sender, editor, **kwargs):
1914     """ log group creation signal """
1915     log_action('CREATE', editor, sender)
1916
1917
1918 def log_group_edit(sender, editor, **kwargs):
1919     """ log group edit signal """
1920     log_action('EDIT', editor, sender)
1921
1922
1923 muddle_user_signals.view_group_created.connect(log_group_create)
1924 muddle_user_signals.view_group_edited.connect(log_group_edit)
1925
1926
1927 def refresh_objects(sender, **kwargs):
1928     """
1929     This was originally the code in the 0009
1930     and then 0010 'force_object_refresh' migration
1931
1932     Force a refresh of all Cluster, Nodes, and VirtualMachines, and
1933     import any new Nodes.
1934     """
1935
1936     if kwargs.get('app', False) and kwargs['app'] == 'ganeti_web':
1937         Cluster.objects.all().update(mtime=None)
1938         Node.objects.all().update(mtime=None)
1939         VirtualMachine.objects.all().update(mtime=None)
1940
1941         write = sys.stdout.write
1942         flush = sys.stdout.flush
1943
1944         def wf(str, newline=False):
1945             if newline:
1946                 write('\n')
1947             write(str)
1948             flush()
1949
1950         wf('- Refresh Cached Cluster Objects')
1951         wf(' > Synchronizing Cluster Nodes ', True)
1952         flush()
1953         for cluster in Cluster.objects.all().iterator():
1954             try:
1955                 cluster.sync_nodes()
1956                 wf('.')
1957             except GanetiApiError:
1958                 wf('E')
1959
1960         wf(' > Refreshing Node Caches ', True)
1961         for node in Node.objects.all().iterator():
1962             try:
1963                 wf('.')
1964             except GanetiApiError:
1965                 wf('E')
1966
1967         wf(' > Refreshing Instance Caches ', True)
1968         for instance in VirtualMachine.objects.all().iterator():
1969             try:
1970                 wf('.')
1971             except GanetiApiError:
1972                 wf('E')
1973         wf('\n')
1974
1975
1976 # Set this as post_migrate hook.
1977 post_migrate.connect(refresh_objects)
1978
1979 # Register permissions on our models.
1980 # These are part of the DB schema and should not be changed without serious
1981 # forethought.
1982 # You *must* syncdb after you change these.
1983 register(permissions.CLUSTER_PARAMS, Cluster, 'ganeti_web')
1984 register(permissions.VIRTUAL_MACHINE_PARAMS, VirtualMachine, 'ganeti_web')
1985
1986
1987 # register log actions
1988 register_log_actions()