ganeti_web/models.py

   1 # coding: utf-8
   2
   3 # Copyright (C) 2010 Oregon State University et al.
   4 # Copyright (C) 2010 Greek Research and Technology Network
   5 #
   6 # This program is free software; you can redistribute it and/or
   7 # modify it under the terms of the GNU General Public License
   8 # as published by the Free Software Foundation; either version 2
   9 # of the License, or (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
  19 # USA.
  20
  21 import binascii
  22 import cPickle
  23 from datetime import datetime, timedelta
  24 from hashlib import sha1
  25 import random
  26 import re
  27 import string
  28 import sys
  29 import time
  30
  31 from django.conf import settings
  32
  33 from django.contrib.auth.models import User, Group
  34 from django.contrib.contenttypes.generic import GenericForeignKey
  35 from django.contrib.contenttypes.models import ContentType
  36 from django.contrib.sites import models as sites_app
  37 from django.contrib.sites.management import create_default_site
  38 from django.core.validators import RegexValidator, MinValueValidator
  39 from django.db import models
  40 from django.db.models import BooleanField, Q, Sum
  41 from django.db.models.query import QuerySet
  42 from django.db.models.signals import post_save, post_syncdb
  43 from django.db.utils import DatabaseError
  44 from django.utils.encoding import force_unicode
  45 from django.utils.translation import ugettext_lazy as _
  46
  47 from django_fields.fields import PickleField
  48
  49 from ganeti_web.logs import register_log_actions
  50
  51 from object_log.models import LogItem
  52 log_action = LogItem.objects.log_action
  53
  54 from object_permissions.registration import register
  55
  56 from muddle_users import signals as muddle_user_signals
  57
  58 from ganeti_web import constants, management, permissions
  59 from ganeti_web.fields import (PatchedEncryptedCharField,
  60                                PreciseDateTimeField, SumIf)
  61 from ganeti_web.util import client
  62 from ganeti_web.util.client import GanetiApiError, REPLACE_DISK_AUTO
  63
  64 from south.signals import post_migrate
  65
  66 if settings.VNC_PROXY:
  67     from ganeti_web.util.vncdaemon.vapclient import (request_forwarding,
  68                                                      request_ssh)
  69
  70
  71 class QuerySetManager(models.Manager):
  72     """
  73     Useful if you want to define manager methods that need to chain. In this
  74     case create a QuerySet class within your model and add all of your methods
  75     directly to the queryset. Example:
  76
  77     class Foo(models.Model):
  78         enabled = fields.BooleanField()
  79         dirty = fields.BooleanField()
  80
  81         class QuerySet:
  82             def active(self):
  83                 return self.filter(enabled=True)
  84             def clean(self):
  85                 return self.filter(dirty=False)
  86
  87     Foo.objects.active().clean()
  88     """
  89
  90     def __getattr__(self, name, *args):
  91         # Cull under/dunder names to avoid certain kinds of recursion. Django
  92         # isn't super-bright here.
  93         if name.startswith('_'):
  94             raise AttributeError
  95         return getattr(self.get_query_set(), name, *args)
  96
  97     def get_query_set(self):
  98         return self.model.QuerySet(self.model)
  99
 100
 101 def generate_random_password(length=12):
 102     "Generate random sequence of specified length"
 103     return "".join(random.sample(string.letters + string.digits, length))
 104
 105 FINISHED_JOBS = 'success', 'unknown', 'error'
 106
 107 RAPI_CACHE = {}
 108 RAPI_CACHE_HASHES = {}
 109
 110
 111 def get_rapi(hash, cluster):
 112     """
 113     Retrieves the cached Ganeti RAPI client for a given hash.  The Hash is
 114     derived from the connection credentials required for a cluster.  If the
 115     client is not yet cached, it will be created and added.
 116
 117     If a hash does not correspond to any cluster then Cluster.DoesNotExist will
 118     be raised.
 119
 120     @param cluster - either a cluster object, or ID of object.  This is used
 121     for resolving the cluster if the client is not already found.  The id is
 122     used rather than the hash, because the hash is mutable.
 123
 124     @return a Ganeti RAPI client.
 125     """
 126     if hash in RAPI_CACHE:
 127         return RAPI_CACHE[hash]
 128
 129     # always look up the instance, even if we were given a Cluster instance
 130     # it ensures we are retrieving the latest credentials.  This helps avoid
 131     # stale credentials.  Retrieve only the values because we don't actually
 132     # need another Cluster instance here.
 133     if isinstance(cluster, (Cluster,)):
 134         cluster = cluster.id
 135     (credentials,) = Cluster.objects.filter(id=cluster) \
 136         .values_list('hash', 'hostname', 'port', 'username', 'password')
 137     hash, host, port, user, password = credentials
 138     user = user or None
 139     # decrypt password
 140     # XXX django-fields only stores str, convert to None if needed
 141     password = Cluster.decrypt_password(password) if password else None
 142     password = None if password in ('None', '') else password
 143
 144     # now that we know hash is fresh, check cache again. The original hash
 145     # could have been stale. This avoids constructing a new RAPI that already
 146     # exists.
 147     if hash in RAPI_CACHE:
 148         return RAPI_CACHE[hash]
 149
 150     # delete any old version of the client that was cached.
 151     if cluster in RAPI_CACHE_HASHES:
 152         del RAPI_CACHE[RAPI_CACHE_HASHES[cluster]]
 153
 154     # Set connect timeout in settings.py so that you do not learn patience.
 155     rapi = client.GanetiRapiClient(host, port, user, password,
 156                                    timeout=settings.RAPI_CONNECT_TIMEOUT)
 157     RAPI_CACHE[hash] = rapi
 158     RAPI_CACHE_HASHES[cluster] = hash
 159     return rapi
 160
 161
 162 def clear_rapi_cache():
 163     """
 164     clears the rapi cache
 165     """
 166     RAPI_CACHE.clear()
 167     RAPI_CACHE_HASHES.clear()
 168
 169
 170 ssh_public_key_re = re.compile(
 171     r'^ssh-(rsa|dsa|dss) [A-Z0-9+/=]+ .+$', re.IGNORECASE)
 172 ssh_public_key_error = _("Enter a valid RSA or DSA SSH key.")
 173 validate_sshkey = RegexValidator(ssh_public_key_re, ssh_public_key_error,
 174                                  "invalid")
 175
 176
 177 class CachedClusterObject(models.Model):
 178     """
 179     Parent class for objects which belong to Ganeti but have cached data in
 180     GWM.
 181
 182     The main point of this class is to permit saving lots of data from Ganeti
 183     so that we don't have to look things up constantly. The Ganeti RAPI is
 184     slow, so avoiding it as much as possible is a good idea.
 185
 186     This class provides transparent caching for all of the data that it
 187     serializes; no explicit cache accesses are required.
 188
 189     This model is abstract and may not be instantiated on its own.
 190     """
 191
 192     serialized_info = models.TextField(default="", editable=False)
 193     mtime = PreciseDateTimeField(null=True, editable=False)
 194     cached = PreciseDateTimeField(null=True, editable=False)
 195     ignore_cache = models.BooleanField(default=False)
 196
 197     last_job_id = None
 198     __info = None
 199     error = None
 200     ctime = None
 201     deleted = False
 202
 203     class Meta:
 204         abstract = True
 205
 206     def save(self, *args, **kwargs):
 207         """
 208         overridden to ensure info is serialized prior to save
 209         """
 210         if not self.serialized_info:
 211             self.serialized_info = cPickle.dumps(self.__info)
 212         super(CachedClusterObject, self).save(*args, **kwargs)
 213
 214     def __init__(self, *args, **kwargs):
 215         super(CachedClusterObject, self).__init__(*args, **kwargs)
 216         self.load_info()
 217
 218     @property
 219     def info(self):
 220         """
 221         A dictionary of metadata for this object.
 222
 223         This is a proxy for the ``serialized_info`` field. Reads from this
 224         property lazily access the field, and writes to this property will be
 225         lazily saved.
 226
 227         Writes to this property do *not* force serialization.
 228         """
 229
 230         if self.__info is None:
 231             if self.serialized_info:
 232                 self.__info = cPickle.loads(str(self.serialized_info))
 233         return self.__info
 234
 235     def _set_info(self, value):
 236         self.__info = value
 237         if value is not None:
 238             self.parse_info()
 239             self.serialized_info = ""
 240
 241     info = info.setter(_set_info)
 242
 243     def load_info(self):
 244         """
 245         Load cached info retrieved from the ganeti cluster.  This function
 246         includes a lazy cache mechanism that uses a timer to decide whether or
 247         not to refresh the cached information with new information from the
 248         ganeti cluster.
 249
 250         This will ignore the cache when self.ignore_cache is True
 251         """
 252
 253         epsilon = timedelta(0, 0, 0, settings.LAZY_CACHE_REFRESH)
 254
 255         if self.id:
 256             if (self.ignore_cache
 257                     or self.cached is None
 258                     or datetime.now() > self.cached + epsilon):
 259                 self.refresh()
 260             elif self.info:
 261                 self.parse_transient_info()
 262             else:
 263                 self.error = 'No Cached Info'
 264
 265     def parse_info(self):
 266         """
 267         Parse all of the attached metadata, and attach it to this object.
 268         """
 269
 270         self.parse_transient_info()
 271         data = self.parse_persistent_info(self.info)
 272         for k in data:
 273             setattr(self, k, data[k])
 274
 275     def refresh(self):
 276         """
 277         Retrieve and parse info from the ganeti cluster.  If successfully
 278         retrieved and parsed, this method will also call save().
 279
 280         If communication with Ganeti fails, an error will be stored in
 281         ``error``.
 282         """
 283
 284         job_data = self.check_job_status()
 285         for k, v in job_data.items():
 286             setattr(self, k, v)
 287
 288         # XXX this try/except is far too big; see if we can pare it down.
 289         try:
 290             info_ = self._refresh()
 291             if info_:
 292                 if info_['mtime']:
 293                     mtime = datetime.fromtimestamp(info_['mtime'])
 294                 else:
 295                     mtime = None
 296                 self.cached = datetime.now()
 297             else:
 298                 # no info retrieved, use current mtime
 299                 mtime = self.mtime
 300
 301             if self.id and (self.mtime is None or mtime > self.mtime):
 302                 # there was an update. Set info and save the object
 303                 self.info = info_
 304                 self.save()
 305             else:
 306                 # There was no change on the server. Only update the cache
 307                 # time. This bypasses the info serialization mechanism and
 308                 # uses a smaller query.
 309                 if job_data:
 310                     self.__class__.objects.filter(pk=self.id) \
 311                         .update(cached=self.cached, **job_data)
 312                 elif self.id is not None:
 313                     self.__class__.objects.filter(pk=self.id) \
 314                         .update(cached=self.cached)
 315
 316         except GanetiApiError, e:
 317             # Use regular expressions to match the quoted message
 318             #  given by GanetiApiError. '\\1' is a group substitution
 319             #  which places the first group '('|\")' in it's place.
 320             comp = re.compile("('|\")(?P<msg>.*)\\1")
 321             err = comp.search(str(e))
 322             # Any search that has 0 results will just return None.
 323             #   That is why we must check for err before proceeding.
 324             if err:
 325                 msg = err.groupdict()['msg']
 326                 self.error = msg
 327             else:
 328                 msg = str(e)
 329                 self.error = str(e)
 330             GanetiError.store_error(msg, obj=self, code=e.code)
 331
 332         else:
 333             if self.error:
 334                 self.error = None
 335                 GanetiError.objects.clear_errors(obj=self)
 336
 337     def _refresh(self):
 338         """
 339         Fetch raw data from the Ganeti cluster.
 340
 341         This must be implemented by children of this class.
 342         """
 343
 344         raise NotImplementedError
 345
 346     def check_job_status(self):
 347         if not self.last_job_id:
 348             return {}
 349
 350         ct = ContentType.objects.get_for_model(self)
 351         qs = Job.objects.filter(content_type=ct, object_id=self.pk)
 352         jobs = qs.order_by("job_id")
 353
 354         updates = {}
 355         op = None
 356         status = 'unknown'
 357
 358         for job in jobs:
 359             try:
 360                 data = self.rapi.GetJobStatus(job.job_id)
 361
 362                 if Job.valid_job(data):
 363                     op = data['ops'][-1]['OP_ID']
 364                     status = data['status']
 365
 366             except GanetiApiError:
 367                 pass
 368
 369             if status in ('success', 'error'):
 370                 for k, v in Job.parse_persistent_info(data).items():
 371                     setattr(job, k, v)
 372
 373             if status == 'unknown':
 374                 job.status = "unknown"
 375                 job.ignore_cache = False
 376
 377             if status in ('success', 'error', 'unknown'):
 378                 _updates = self._complete_job(self.cluster_id,
 379                                               self.hostname, op, status)
 380                 # XXX if the delete flag is set in updates then delete this
 381                 # model this happens here because _complete_job cannot delete
 382                 # this model
 383                 if _updates:
 384                     if 'deleted' in _updates:
 385                         # Delete ourselves. Also delete the job that caused us
 386                         # to delete ourselves; see #8439 for "fun" details.
 387                         # Order matters; the job's deletion cascades over us.
 388                         # Revisit that when we finally nuke all this caching
 389                         # bullshit.
 390                         self.delete()
 391                         job.delete()
 392                     else:
 393                         updates.update(_updates)
 394
 395         # we only care about the very last job for resetting the cache flags
 396         if not jobs or status in ('success', 'error', 'unknown'):
 397             updates['ignore_cache'] = False
 398             updates['last_job'] = None
 399
 400         return updates
 401
 402     @classmethod
 403     def _complete_job(cls, cluster_id, hostname, op, status):
 404         """
 405         Process a completed job.  This method will make any updates to related
 406         classes (like deleting an instance template) and return any data that
 407         should be updated.  This is a class method so that this processing can
 408         be done without a full instance.
 409
 410         @returns dict of updated values
 411         """
 412
 413         pass
 414
 415     def parse_transient_info(self):
 416         """
 417         Parse properties from cached info that is stored on the class but not
 418         in the database.
 419
 420         These properties will be loaded every time the object is instantiated.
 421         Properties stored on the class cannot be search efficiently via the
 422         django query api.
 423
 424         This method is specific to the child object.
 425         """
 426
 427         info_ = self.info
 428         # XXX ganeti 2.1 ctime is always None
 429         # XXX this means that we could nuke the conditionals!
 430         if info_['ctime'] is not None:
 431             self.ctime = datetime.fromtimestamp(info_['ctime'])
 432
 433     @classmethod
 434     def parse_persistent_info(cls, info):
 435         """
 436         Parse properties from cached info that are stored in the database.
 437
 438         These properties will be searchable by the django query api.
 439
 440         This method is specific to the child object.
 441         """
 442
 443         # mtime is sometimes None if object has never been modified
 444         if info['mtime'] is None:
 445             return {'mtime': None}
 446         return {'mtime': datetime.fromtimestamp(info['mtime'])}
 447
 448
 449 class JobManager(models.Manager):
 450     """
 451     Custom manager for Ganeti Jobs model
 452     """
 453     def create(self, **kwargs):
 454         """ helper method for creating a job with disabled cache """
 455         job = Job(ignore_cache=True, **kwargs)
 456         job.save(force_insert=True)
 457         return job
 458
 459
 460 class Job(CachedClusterObject):
 461     """
 462     model representing a job being run on a ganeti Cluster.  This includes
 463     operations such as creating or delting a virtual machine.
 464
 465     Jobs are a special type of CachedClusterObject.  Job's run once then become
 466     immutable.  The lazy cache is modified to become permanent once a complete
 467     status (success/error) has been detected.  The cache can be disabled by
 468     settning ignore_cache=True.
 469     """
 470
 471     job_id = models.IntegerField()
 472     content_type = models.ForeignKey(ContentType, related_name="+")
 473     object_id = models.IntegerField()
 474     obj = GenericForeignKey('content_type', 'object_id')
 475     cluster = models.ForeignKey('Cluster', related_name='jobs', editable=False)
 476     cluster_hash = models.CharField(max_length=40, editable=False)
 477
 478     finished = models.DateTimeField(null=True, blank=True)
 479     status = models.CharField(max_length=10)
 480     op = models.CharField(max_length=50)
 481
 482     objects = JobManager()
 483
 484     def save(self, *args, **kwargs):
 485         """
 486         sets the cluster_hash for newly saved instances
 487         """
 488         if self.id is None or self.cluster_hash == '':
 489             self.cluster_hash = self.cluster.hash
 490
 491         super(Job, self).save(*args, **kwargs)
 492
 493     @models.permalink
 494     def get_absolute_url(self):
 495         job = '%s/job/(?P<job_id>\d+)' % self.cluster
 496
 497         return 'ganeti_web.views.jobs.detail', (), {'job': job}
 498
 499     @property
 500     def rapi(self):
 501         return get_rapi(self.cluster_hash, self.cluster_id)
 502
 503     def _refresh(self):
 504         return self.rapi.GetJobStatus(self.job_id)
 505
 506     def load_info(self):
 507         """
 508         Load info for class.  This will load from ganeti if ignore_cache==True,
 509         otherwise this will always load from the cache.
 510         """
 511         if self.id and (self.ignore_cache or self.info is None):
 512             try:
 513                 self.refresh()
 514             except GanetiApiError, e:
 515                 # if the Job has been archived then we don't know whether it
 516                 # was successful or not. Mark it as unknown.
 517                 if e.code == 404:
 518                     self.status = 'unknown'
 519                     self.save()
 520                 else:
 521                     # its possible the cluster or crednetials are bad. fail
 522                     # silently
 523                     pass
 524
 525     def refresh(self):
 526         self.info = self._refresh()
 527         valid = self.valid_job(self.info)
 528         if valid:
 529             self.save()
 530         # else:
 531         #     Job.objects.get(job_id=self.info['id']).delete()
 532
 533     @classmethod
 534     def valid_job(cls, info):
 535         status = info.get('status')
 536         ops = info.get('ops')
 537         return not (ops is None and status is None)
 538
 539     @classmethod
 540     def parse_op(cls, info):
 541         ops = info['ops']
 542         op = None
 543         if ops:
 544             # Return the most recent operation
 545             op = ops[-1]['OP_ID']
 546         return op
 547
 548     @classmethod
 549     def parse_persistent_info(cls, info):
 550         """
 551         Parse status and turn off cache bypass flag if job has finished
 552         """
 553         if not cls.valid_job():
 554             return
 555         op = cls.parse_op(info)
 556         data = {'status': info['status'], 'op': op}
 557         if data['status'] in ('error', 'success'):
 558             data['ignore_cache'] = False
 559         if info['end_ts']:
 560             data['finished'] = cls.parse_end_timestamp(info)
 561         return data
 562
 563     @staticmethod
 564     def parse_end_timestamp(info):
 565         sec, micro = info['end_ts']
 566         return datetime.fromtimestamp(sec + (micro / 1000000.0))
 567
 568     def parse_transient_info(self):
 569         pass
 570
 571     @property
 572     def current_operation(self):
 573         """
 574         Jobs may consist of multiple commands/operations.  This helper
 575         method will return the operation that is currently running or errored
 576         out, or the last operation if all operations have completed
 577
 578         @returns raw name of the current operation
 579         """
 580         info = self.info
 581         index = 0
 582         for i in range(len(info['opstatus'])):
 583             if info['opstatus'][i] != 'success':
 584                 index = i
 585                 break
 586         return info['ops'][index]['OP_ID']
 587
 588     @property
 589     def operation(self):
 590         """
 591         Returns the last operation, which is generally the primary operation.
 592         """
 593         return self.parse_op(self.info)
 594
 595     def __repr__(self):
 596         return "<Job %d (%d), status %r>" % (self.id, self.job_id,
 597                                              self.status)
 598
 599     __unicode__ = __repr__
 600
 601
 602 class VirtualMachine(CachedClusterObject):
 603     """
 604     The VirtualMachine (VM) model represents VMs within a Ganeti cluster.
 605
 606     The majority of properties are a cache for data stored in the cluster.
 607     All data retrieved via the RAPI is stored in VirtualMachine.info, and
 608     serialized automatically into VirtualMachine.serialized_info.
 609
 610     Attributes that need to be searchable should be stored as model fields.
 611     All other attributes will be stored within VirtualMachine.info.
 612
 613     This object uses a lazy update mechanism on instantiation.  If the cached
 614     info from the Ganeti cluster has expired, it will trigger an update.  This
 615     allows the cache to function in the absence of a periodic update mechanism
 616     such as Cron, Celery, or Threads.
 617
 618     XXX Serialized_info can possibly be changed to a CharField if an upper
 619         limit can be determined. (Later Date, if it will optimize db)
 620
 621     """
 622     cluster = models.ForeignKey('Cluster', related_name='virtual_machines',
 623                                 editable=False, default=0)
 624     hostname = models.CharField(max_length=128, db_index=True)
 625     owner = models.ForeignKey('ClusterUser', related_name='virtual_machines',
 626                               null=True, blank=True,
 627                               on_delete=models.SET_NULL)
 628     virtual_cpus = models.IntegerField(default=-1)
 629     disk_size = models.IntegerField(default=-1)
 630     ram = models.IntegerField(default=-1)
 631     minram = models.IntegerField(default=-1)
 632     cluster_hash = models.CharField(max_length=40, editable=False)
 633     operating_system = models.CharField(max_length=128)
 634     status = models.CharField(max_length=14)
 635
 636     # node relations
 637     primary_node = models.ForeignKey('Node', related_name='primary_vms',
 638                                      null=True, blank=True)
 639     secondary_node = models.ForeignKey('Node', related_name='secondary_vms',
 640                                        null=True, blank=True)
 641
 642     # The last job reference indicates that there is at least one pending job
 643     # for this virtual machine.  There may be more than one job, and that can
 644     # never be prevented.  This just indicates that job(s) are pending and the
 645     # job related code should be run (status, cleanup, etc).
 646     last_job = models.ForeignKey('Job', related_name="+", null=True,
 647                                  blank=True)
 648
 649     # deleted flag indicates a VM is being deleted, but the job has not
 650     # completed yet.  VMs that have pending_delete are still displayed in lists
 651     # and counted in quotas, but only so status can be checked.
 652     pending_delete = models.BooleanField(default=False)
 653     deleted = False
 654
 655     # Template temporarily stores parameters used to create this virtual
 656     # machine. This template is used to recreate the values entered into the
 657     # form.
 658     template = models.ForeignKey("VirtualMachineTemplate",
 659                                  related_name="instances", null=True,
 660                                  blank=True)
 661
 662     class Meta:
 663         ordering = ["hostname"]
 664         unique_together = (("cluster", "hostname"),)
 665
 666     def __unicode__(self):
 667         return self.hostname
 668
 669     def save(self, *args, **kwargs):
 670         """
 671         sets the cluster_hash for newly saved instances
 672         """
 673         if self.id is None:
 674             self.cluster_hash = self.cluster.hash
 675
 676         info_ = self.info
 677         if info_:
 678             found = False
 679             remove = []
 680             if self.cluster.username:
 681                 for tag in info_['tags']:
 682                     # Update owner Tag. Make sure the tag is set to the owner
 683                     #  that is set in webmgr.
 684                     if tag.startswith(constants.OWNER_TAG):
 685                         id = int(tag[len(constants.OWNER_TAG):])
 686                         # Since there is no 'update tag' delete old tag and
 687                         #  replace with tag containing correct owner id.
 688                         if id == self.owner_id:
 689                             found = True
 690                         else:
 691                             remove.append(tag)
 692                 if remove:
 693                     self.rapi.DeleteInstanceTags(self.hostname, remove)
 694                     for tag in remove:
 695                         info_['tags'].remove(tag)
 696                 if self.owner_id and not found:
 697                     tag = '%s%s' % (constants.OWNER_TAG, self.owner_id)
 698                     self.rapi.AddInstanceTags(self.hostname, [tag])
 699                     self.info['tags'].append(tag)
 700
 701         super(VirtualMachine, self).save(*args, **kwargs)
 702
 703     @models.permalink
 704     def get_absolute_url(self):
 705         """
 706         Return absolute url for this instance.
 707         """
 708
 709         return 'instance-detail', (), {'cluster_slug': self.cluster.slug,
 710                                        'instance': self.hostname}
 711
 712     @property
 713     def rapi(self):
 714         return get_rapi(self.cluster_hash, self.cluster_id)
 715
 716     @property
 717     def is_running(self):
 718         return self.status == 'running'
 719
 720     @classmethod
 721     def parse_persistent_info(cls, info):
 722         """
 723         Loads all values from cached info, included persistent properties that
 724         are stored in the database
 725         """
 726         data = super(VirtualMachine, cls).parse_persistent_info(info)
 727
 728         # Parse resource properties
 729         data['ram'] = info['beparams']['memory']
 730         data['virtual_cpus'] = info['beparams']['vcpus']
 731         # Sum up the size of each disk used by the VM
 732         disk_size = 0
 733         for disk in info['disk.sizes']:
 734             disk_size += disk
 735         data['disk_size'] = disk_size
 736         data['operating_system'] = info['os']
 737         data['status'] = info['status']
 738
 739         primary = info['pnode']
 740         if primary:
 741             try:
 742                 data['primary_node'] = Node.objects.get(hostname=primary)
 743             except Node.DoesNotExist:
 744                 # node is not created yet.  fail silently
 745                 data['primary_node'] = None
 746         else:
 747             data['primary_node'] = None
 748
 749         secondary = info['snodes']
 750         if len(secondary):
 751             secondary = secondary[0]
 752             try:
 753                 data['secondary_node'] = Node.objects.get(hostname=secondary)
 754             except Node.DoesNotExist:
 755                 # node is not created yet.  fail silently
 756                 data['secondary_node'] = None
 757         else:
 758             data['secondary_node'] = None
 759
 760         return data
 761
 762     @classmethod
 763     def _complete_job(cls, cluster_id, hostname, op, status):
 764         """
 765         if the cache bypass is enabled then check the status of the last job
 766         when the job is complete we can reenable the cache.
 767
 768         @returns - dictionary of values that were updates
 769         """
 770
 771         if status == 'unknown':
 772             # unknown status, the job was archived before it's final status
 773             # was polled.  Impossible to tell what happened.  Clear the job
 774             # so it is no longer polled.
 775             #
 776             # XXX This VM might be added by the CLI and be in an invalid
 777             # pending_delete state.  clearing pending_delete prevents this
 778             # but will result in "missing" vms in some cases.
 779             return dict(pending_delete=False)
 780
 781         base = VirtualMachine.objects.filter(cluster=cluster_id,
 782                                              hostname=hostname)
 783         if op == 'OP_INSTANCE_REMOVE':
 784             if status == 'success':
 785                 # XXX can't actually delete here since it would cause a
 786                 # recursive loop
 787                 return dict(deleted=True)
 788
 789         elif op == 'OP_INSTANCE_CREATE' and status == 'success':
 790             # XXX must update before deleting the template to maintain
 791             # referential integrity.  as a consequence return no other
 792             # updates.
 793             base.update(template=None)
 794             VirtualMachineTemplate.objects \
 795                 .filter(instances__hostname=hostname,
 796                         instances__cluster=cluster_id) \
 797                 .delete()
 798             return dict(template=None)
 799         return
 800
 801     def _refresh(self):
 802         # XXX if delete is pending then no need to refresh this object.
 803         if self.pending_delete or self.template_id:
 804             return None
 805         return self.rapi.GetInstance(self.hostname)
 806
 807     def shutdown(self, timeout=None):
 808         if timeout is None:
 809             id = self.rapi.ShutdownInstance(self.hostname)
 810         else:
 811             id = self.rapi.ShutdownInstance(self.hostname, timeout=timeout)
 812
 813         job = Job.objects.create(job_id=id, obj=self,
 814                                  cluster_id=self.cluster_id)
 815         self.last_job = job
 816         VirtualMachine.objects.filter(pk=self.id) \
 817             .update(last_job=job, ignore_cache=True)
 818         return job
 819
 820     def startup(self):
 821         id = self.rapi.StartupInstance(self.hostname)
 822         job = Job.objects.create(job_id=id, obj=self,
 823                                  cluster_id=self.cluster_id)
 824         self.last_job = job
 825         VirtualMachine.objects.filter(pk=self.id) \
 826             .update(last_job=job, ignore_cache=True)
 827         return job
 828
 829     def reboot(self):
 830         id = self.rapi.RebootInstance(self.hostname)
 831         job = Job.objects.create(job_id=id, obj=self,
 832                                  cluster_id=self.cluster_id)
 833         self.last_job = job
 834         VirtualMachine.objects.filter(pk=self.id) \
 835             .update(last_job=job, ignore_cache=True)
 836         return job
 837
 838     def migrate(self, mode='live', cleanup=False):
 839         """
 840         Migrates this VirtualMachine to another node.
 841
 842         Only works if the disk type is DRDB.
 843
 844         @param mode: live or non-live
 845         @param cleanup: clean up a previous migration, default is False
 846         """
 847         id = self.rapi.MigrateInstance(self.hostname, mode, cleanup)
 848         job = Job.objects.create(job_id=id, obj=self,
 849                                  cluster_id=self.cluster_id)
 850         self.last_job = job
 851         VirtualMachine.objects.filter(pk=self.id) \
 852             .update(last_job=job, ignore_cache=True)
 853         return job
 854
 855     def replace_disks(self, mode=REPLACE_DISK_AUTO, disks=None, node=None,
 856                       iallocator=None):
 857         id = self.rapi.ReplaceInstanceDisks(self.hostname, disks, mode, node,
 858                                             iallocator)
 859         job = Job.objects.create(job_id=id, obj=self,
 860                                  cluster_id=self.cluster_id)
 861         self.last_job = job
 862         VirtualMachine.objects.filter(pk=self.id) \
 863             .update(last_job=job, ignore_cache=True)
 864         return job
 865
 866     def setup_ssh_forwarding(self, sport=0):
 867         """
 868         Poke a proxy to start SSH forwarding.
 869
 870         Returns None if no proxy is configured, or if there was an error
 871         contacting the proxy.
 872         """
 873
 874         command = self.rapi.GetInstanceConsole(self.hostname)["command"]
 875
 876         if settings.VNC_PROXY:
 877             proxy_server = settings.VNC_PROXY.split(":")
 878             password = generate_random_password()
 879             sport = request_ssh(proxy_server, sport, self.info["pnode"],
 880                                 self.info["network_port"], password, command)
 881
 882             if sport:
 883                 return proxy_server[0], sport, password
 884
 885     def setup_vnc_forwarding(self, sport=0, tls=False):
 886         """
 887         Obtain VNC forwarding information, optionally configuring a proxy.
 888
 889         Returns None if a proxy is configured and there was an error
 890         contacting the proxy.
 891         """
 892
 893         password = ''
 894         info_ = self.info
 895         port = info_['network_port']
 896         node = info_['pnode']
 897
 898         # use proxy for VNC connection
 899         if settings.VNC_PROXY:
 900             proxy_server = settings.VNC_PROXY.split(":")
 901             password = generate_random_password()
 902             result = request_forwarding(proxy_server, node, port, password,
 903                                         sport=sport, tls=tls)
 904             if result:
 905                 return proxy_server[0], int(result), password
 906         else:
 907             return node, port, password
 908
 909     def __repr__(self):
 910         return "<VirtualMachine: '%s'>" % self.hostname
 911
 912
 913 class Node(CachedClusterObject):
 914     """
 915     The Node model represents nodes within a Ganeti cluster.
 916
 917     The majority of properties are a cache for data stored in the cluster.
 918     All data retrieved via the RAPI is stored in VirtualMachine.info, and
 919     serialized automatically into VirtualMachine.serialized_info.
 920
 921     Attributes that need to be searchable should be stored as model fields.
 922     All other attributes will be stored within VirtualMachine.info.
 923     """
 924
 925     ROLE_CHOICES = ((k, v) for k, v in constants.NODE_ROLE_MAP.items())
 926
 927     cluster = models.ForeignKey('Cluster', related_name='nodes')
 928     hostname = models.CharField(max_length=128, unique=True)
 929     cluster_hash = models.CharField(max_length=40, editable=False)
 930     offline = models.BooleanField()
 931     role = models.CharField(max_length=1, choices=ROLE_CHOICES)
 932     ram_total = models.IntegerField(default=-1)
 933     ram_free = models.IntegerField(default=-1)
 934     disk_total = models.IntegerField(default=-1)
 935     disk_free = models.IntegerField(default=-1)
 936     cpus = models.IntegerField(null=True, blank=True)
 937
 938     # The last job reference indicates that there is at least one pending job
 939     # for this virtual machine.  There may be more than one job, and that can
 940     # never be prevented.  This just indicates that job(s) are pending and the
 941     # job related code should be run (status, cleanup, etc).
 942     last_job = models.ForeignKey('Job', related_name="+", null=True,
 943                                  blank=True)
 944
 945     def __unicode__(self):
 946         return self.hostname
 947
 948     def save(self, *args, **kwargs):
 949         """
 950         sets the cluster_hash for newly saved instances
 951         """
 952         if self.id is None:
 953             self.cluster_hash = self.cluster.hash
 954         super(Node, self).save(*args, **kwargs)
 955
 956     @models.permalink
 957     def get_absolute_url(self):
 958         """
 959         Return absolute url for this node.
 960         """
 961
 962         return 'node-detail', (), {'cluster_slug': self.cluster.slug,
 963                                    'host': self.hostname}
 964
 965     def _refresh(self):
 966         """ returns node info from the ganeti server """
 967         return self.rapi.GetNode(self.hostname)
 968
 969     @property
 970     def rapi(self):
 971         return get_rapi(self.cluster_hash, self.cluster_id)
 972
 973     @classmethod
 974     def parse_persistent_info(cls, info):
 975         """
 976         Loads all values from cached info, included persistent properties that
 977         are stored in the database
 978         """
 979         data = super(Node, cls).parse_persistent_info(info)
 980
 981         # Parse resource properties
 982         data['ram_total'] = info.get("mtotal") or 0
 983         data['ram_free'] = info.get("mfree") or 0
 984         data['disk_total'] = info.get("dtotal") or 0
 985         data['disk_free'] = info.get("dfree") or 0
 986         data['cpus'] = info.get("csockets")
 987         data['offline'] = info['offline']
 988         data['role'] = info['role']
 989         return data
 990
 991     @property
 992     def ram(self):
 993         """ returns dict of free and total ram """
 994         values = VirtualMachine.objects \
 995             .filter(Q(primary_node=self) | Q(secondary_node=self)) \
 996             .filter(status='running') \
 997             .exclude(ram=-1).order_by() \
 998             .aggregate(used=Sum('ram'))
 999
1000         total = self.ram_total
1001         used = total - self.ram_free
1002         allocated = values.get("used") or 0
1003         free = total - allocated if allocated >= 0 and total >= 0 else -1
1004
1005         return {
1006             'total': total,
1007             'free':  free,
1008             'allocated': allocated,
1009             'used': used,
1010         }
1011
1012     @property
1013     def disk(self):
1014         """ returns dict of free and total disk space """
1015         values = VirtualMachine.objects \
1016             .filter(Q(primary_node=self) | Q(secondary_node=self)) \
1017             .exclude(disk_size=-1).order_by() \
1018             .aggregate(used=Sum('disk_size'))
1019
1020         total = self.disk_total
1021         used = total - self.disk_free
1022         allocated = values.get("used") or 0
1023         free = total - allocated if allocated >= 0 and total >= 0 else -1
1024
1025         return {
1026             'total': total,
1027             'free':  free,
1028             'allocated': allocated,
1029             'used': used,
1030         }
1031
1032     @property
1033     def allocated_cpus(self):
1034         values = VirtualMachine.objects \
1035             .filter(primary_node=self, status='running') \
1036             .exclude(virtual_cpus=-1).order_by() \
1037             .aggregate(cpus=Sum('virtual_cpus'))
1038         return values.get("cpus") or 0
1039
1040     def set_role(self, role, force=False):
1041         """
1042         Sets the role for this node
1043
1044         @param role - one of the following choices:
1045             * master
1046             * master-candidate
1047             * regular
1048             * drained
1049             * offline
1050         """
1051         id = self.rapi.SetNodeRole(self.hostname, role, force)
1052         job = Job.objects.create(job_id=id, obj=self,
1053                                  cluster_id=self.cluster_id)
1054         self.last_job = job
1055         Node.objects.filter(pk=self.pk).update(ignore_cache=True, last_job=job)
1056         return job
1057
1058     def evacuate(self, iallocator=None, node=None):
1059         """
1060         migrates all secondary instances off this node
1061         """
1062         id = self.rapi.EvacuateNode(self.hostname, iallocator=iallocator,
1063                                     remote_node=node)
1064         job = Job.objects.create(job_id=id, obj=self,
1065                                  cluster_id=self.cluster_id)
1066         self.last_job = job
1067         Node.objects.filter(pk=self.pk) \
1068             .update(ignore_cache=True, last_job=job)
1069         return job
1070
1071     def migrate(self, mode=None):
1072         """
1073         migrates all primary instances off this node
1074         """
1075         id = self.rapi.MigrateNode(self.hostname, mode)
1076         job = Job.objects.create(job_id=id, obj=self,
1077                                  cluster_id=self.cluster_id)
1078         self.last_job = job
1079         Node.objects.filter(pk=self.pk).update(ignore_cache=True, last_job=job)
1080         return job
1081
1082     def __repr__(self):
1083         return "<Node: '%s'>" % self.hostname
1084
1085
1086 class Cluster(CachedClusterObject):
1087     """
1088     A Ganeti cluster that is being tracked by this manager tool
1089     """
1090     hostname = models.CharField(_('hostname'), max_length=128, unique=True)
1091     slug = models.SlugField(_('slug'), max_length=50, unique=True,
1092                             db_index=True)
1093     port = models.PositiveIntegerField(_('port'), default=5080)
1094     description = models.CharField(_('description'), max_length=128,
1095                                    blank=True)
1096     username = models.CharField(_('username'), max_length=128, blank=True)
1097     password = PatchedEncryptedCharField(_('password'), default="",
1098                                          max_length=128, blank=True)
1099     hash = models.CharField(_('hash'), max_length=40, editable=False)
1100
1101     # quota properties
1102     virtual_cpus = models.IntegerField(_('Virtual CPUs'), null=True,
1103                                        blank=True)
1104     disk = models.IntegerField(_('disk'), null=True, blank=True)
1105     ram = models.IntegerField(_('ram'), null=True, blank=True)
1106
1107     # The last job reference indicates that there is at least one pending job
1108     # for this virtual machine.  There may be more than one job, and that can
1109     # never be prevented.  This just indicates that job(s) are pending and the
1110     # job related code should be run (status, cleanup, etc).
1111     last_job = models.ForeignKey('Job', related_name='cluster_last_job',
1112                                  null=True, blank=True)
1113
1114     class Meta:
1115         ordering = ["hostname", "description"]
1116
1117     def __unicode__(self):
1118         return self.hostname
1119
1120     def save(self, *args, **kwargs):
1121         self.hash = self.create_hash()
1122         super(Cluster, self).save(*args, **kwargs)
1123
1124     @models.permalink
1125     def get_absolute_url(self):
1126         return 'cluster-detail', (), {'cluster_slug': self.slug}
1127
1128     # XXX probably hax
1129     @property
1130     def cluster_id(self):
1131         return self.id
1132
1133     @classmethod
1134     def decrypt_password(cls, value):
1135         """
1136         Convenience method for decrypting a password without an instance.
1137         This was partly cribbed from django-fields which only allows decrypting
1138         from a model instance.
1139
1140         If the password appears to be encrypted, this method will decrypt it;
1141         otherwise, it will return the password unchanged.
1142
1143         This method is bonghits.
1144         """
1145
1146         field, chaff, chaff, chaff = cls._meta.get_field_by_name('password')
1147
1148         if value.startswith(field.prefix):
1149             ciphertext = value[len(field.prefix):]
1150             plaintext = field.cipher.decrypt(binascii.a2b_hex(ciphertext))
1151             password = plaintext.split('\0')[0]
1152         else:
1153             password = value
1154
1155         return force_unicode(password)
1156
1157     @property
1158     def rapi(self):
1159         """
1160         retrieves the rapi client for this cluster.
1161         """
1162         # XXX always pass self in.  not only does it avoid querying this object
1163         # from the DB a second time, it also prevents a recursion loop caused
1164         # by __init__ fetching info from the Cluster
1165         return get_rapi(self.hash, self)
1166
1167     def create_hash(self):
1168         """
1169         Creates a hash for this cluster based on credentials required for
1170         connecting to the server
1171         """
1172         s = '%s%s%s%s' % (self.username, self.password, self.hostname,
1173                           self.port)
1174         return sha1(s).hexdigest()
1175
1176     def get_default_quota(self):
1177         """
1178         Returns the default quota for this cluster
1179         """
1180         return {
1181             "default": 1,
1182             "ram": self.ram,
1183             "disk": self.disk,
1184             "virtual_cpus": self.virtual_cpus,
1185         }
1186
1187     def get_quota(self, user=None):
1188         """
1189         Get the quota for a ClusterUser
1190
1191         @return user's quota, default quota, or none
1192         """
1193         if user is None:
1194             return self.get_default_quota()
1195
1196         # attempt to query user specific quota first. if it does not exist
1197         # then fall back to the default quota
1198         query = Quota.objects.filter(cluster=self, user=user)
1199         quotas = query.values('ram', 'disk', 'virtual_cpus')
1200         if quotas:
1201             quota = quotas[0]
1202             quota['default'] = 0
1203             return quota
1204
1205         return self.get_default_quota()
1206
1207     def set_quota(self, user, data):
1208         """
1209         Set the quota for a ClusterUser.
1210
1211         If data is None, the quota will be removed.
1212
1213         @param values: dictionary of values, or None to delete the quota
1214         """
1215
1216         kwargs = {'cluster': self, 'user': user}
1217         if data is None:
1218             Quota.objects.filter(**kwargs).delete()
1219         else:
1220             quota, new = Quota.objects.get_or_create(**kwargs)
1221             quota.__dict__.update(data)
1222             quota.save()
1223
1224     @classmethod
1225     def get_quotas(cls, clusters=None, user=None):
1226         """ retrieve a bulk list of cluster quotas """
1227
1228         if clusters is None:
1229             clusters = Cluster.objects.all()
1230
1231         quotas = {}
1232         cluster_id_map = {}
1233         for cluster in clusters:
1234             quotas[cluster] = {
1235                 'default': 1,
1236                 'ram': cluster.ram,
1237                 'disk': cluster.disk,
1238                 'virtual_cpus': cluster.virtual_cpus,
1239             }
1240             cluster_id_map[cluster.id] = cluster
1241
1242         # get user's custom queries if any
1243         if user is not None:
1244             qs = Quota.objects.filter(cluster__in=clusters, user=user)
1245             values = qs.values('ram', 'disk', 'virtual_cpus', 'cluster__id')
1246
1247             for custom in values:
1248                 try:
1249                     cluster = cluster_id_map[custom['cluster__id']]
1250                 except KeyError:
1251                     continue
1252                 custom['default'] = 0
1253                 del custom['cluster__id']
1254                 quotas[cluster] = custom
1255
1256         return quotas
1257
1258     def sync_virtual_machines(self, remove=False):
1259         """
1260         Synchronizes the VirtualMachines in the database with the information
1261         this ganeti cluster has:
1262             * VMs no longer in ganeti are deleted
1263             * VMs missing from the database are added
1264         """
1265         ganeti = self.instances()
1266         db = self.virtual_machines.all().values_list('hostname', flat=True)
1267
1268         # add VMs missing from the database
1269         for hostname in filter(lambda x: unicode(x) not in db, ganeti):
1270             vm = VirtualMachine.objects.create(cluster=self, hostname=hostname)
1271             vm.refresh()
1272
1273         # deletes VMs that are no longer in ganeti
1274         if remove:
1275             missing_ganeti = filter(lambda x: str(x) not in ganeti, db)
1276             if missing_ganeti:
1277                 self.virtual_machines \
1278                     .filter(hostname__in=missing_ganeti).delete()
1279
1280     def sync_nodes(self, remove=False):
1281         """
1282         Synchronizes the Nodes in the database with the information
1283         this ganeti cluster has:
1284             * Nodes no longer in ganeti are deleted
1285             * Nodes missing from the database are added
1286         """
1287         ganeti = self.rapi.GetNodes()
1288         db = self.nodes.all().values_list('hostname', flat=True)
1289
1290         # add Nodes missing from the database
1291         for hostname in filter(lambda x: unicode(x) not in db, ganeti):
1292             node = Node.objects.create(cluster=self, hostname=hostname)
1293             node.refresh()
1294
1295         # deletes Nodes that are no longer in ganeti
1296         if remove:
1297             missing_ganeti = filter(lambda x: str(x) not in ganeti, db)
1298             if missing_ganeti:
1299                 self.nodes.filter(hostname__in=missing_ganeti).delete()
1300
1301     @property
1302     def missing_in_ganeti(self):
1303         """
1304         Returns a list of VirtualMachines that are missing from the Ganeti
1305         cluster but present in the database.
1306         """
1307         ganeti = self.instances()
1308         qs = self.virtual_machines.exclude(template__isnull=False)
1309         db = qs.values_list('hostname', flat=True)
1310         return [x for x in db if str(x) not in ganeti]
1311
1312     @property
1313     def missing_in_db(self):
1314         """
1315         Returns list of VirtualMachines that are missing from the database, but
1316         present in ganeti
1317         """
1318         ganeti = self.instances()
1319         db = self.virtual_machines.all().values_list('hostname', flat=True)
1320         return [x for x in ganeti if unicode(x) not in db]
1321
1322     @property
1323     def nodes_missing_in_db(self):
1324         """
1325         Returns list of Nodes that are missing from the database, but present
1326         in ganeti.
1327         """
1328         try:
1329             ganeti = self.rapi.GetNodes()
1330         except GanetiApiError:
1331             ganeti = []
1332         db = self.nodes.all().values_list('hostname', flat=True)
1333         return [x for x in ganeti if unicode(x) not in db]
1334
1335     @property
1336     def nodes_missing_in_ganeti(self):
1337         """
1338         Returns list of Nodes that are missing from the ganeti cluster
1339         but present in the database
1340         """
1341         try:
1342             ganeti = self.rapi.GetNodes()
1343         except GanetiApiError:
1344             ganeti = []
1345         db = self.nodes.all().values_list('hostname', flat=True)
1346         return filter(lambda x: str(x) not in ganeti, db)
1347
1348     @property
1349     def available_ram(self):
1350         """ returns dict of free and total ram """
1351         nodes = self.nodes.exclude(ram_total=-1) \
1352             .aggregate(total=Sum('ram_total'), free=Sum('ram_free'))
1353         total = max(nodes.get("total", 0), 0)
1354         free = max(nodes.get("free", 0), 0)
1355         used = total - free
1356         values = self.virtual_machines \
1357             .filter(status='running') \
1358             .exclude(ram=-1).order_by() \
1359             .aggregate(used=Sum('ram'))
1360
1361         if values.get("used") is None:
1362             allocated = 0
1363         else:
1364             allocated = values["used"]
1365
1366         free = max(total - allocated, 0)
1367
1368         return {
1369             'total': total,
1370             'free': free,
1371             'allocated': allocated,
1372             'used': used,
1373         }
1374
1375     @property
1376     def available_disk(self):
1377         """ returns dict of free and total disk space """
1378         nodes = self.nodes.exclude(disk_total=-1) \
1379             .aggregate(total=Sum('disk_total'), free=Sum('disk_free'))
1380         total = max(nodes.get("total", 0), 0)
1381         free = max(nodes.get("free", 0), 0)
1382         used = total - free
1383         values = self.virtual_machines \
1384             .exclude(disk_size=-1).order_by() \
1385             .aggregate(used=Sum('disk_size'))
1386
1387         if values.get("used") is None:
1388             allocated = 0
1389         else:
1390             allocated = values["used"]
1391
1392         free = max(total - allocated, 0)
1393
1394         return {
1395             'total': total,
1396             'free': free,
1397             'allocated': allocated,
1398             'used': used,
1399         }
1400
1401     def _refresh(self):
1402         return self.rapi.GetInfo()
1403
1404     def instances(self, bulk=False):
1405         """Gets all VMs which reside under the Cluster
1406         Calls the rapi client for all instances.
1407         """
1408         try:
1409             return self.rapi.GetInstances(bulk=bulk)
1410         except GanetiApiError:
1411             return []
1412
1413     def instance(self, instance):
1414         """Get a single Instance
1415         Calls the rapi client for a specific instance.
1416         """
1417         try:
1418             return self.rapi.GetInstance(instance)
1419         except GanetiApiError:
1420             return None
1421
1422     def redistribute_config(self):
1423         """
1424         Redistribute config from cluster's master node to all
1425         other nodes.
1426         """
1427         # no exception handling, because it's being done in a view
1428         id = self.rapi.RedistributeConfig()
1429         job = Job.objects.create(job_id=id, obj=self, cluster_id=self.id)
1430         self.last_job = job
1431         Cluster.objects.filter(pk=self.id) \
1432             .update(last_job=job, ignore_cache=True)
1433         return job
1434
1435
1436 class VirtualMachineTemplate(models.Model):
1437     """
1438     Virtual Machine Template holds all the values for the create virtual
1439     machine form so that they can automatically be used or edited by a user.
1440     """
1441
1442     template_name = models.CharField(max_length=255, default="")
1443     temporary = BooleanField(verbose_name=_("Temporary"), default=False)
1444     description = models.CharField(max_length=255, default="")
1445     cluster = models.ForeignKey(Cluster, related_name="templates", null=True,
1446                                 blank=True)
1447     start = models.BooleanField(verbose_name=_('Start up After Creation'),
1448                                 default=True)
1449     no_install = models.BooleanField(verbose_name=_('Do not install OS'),
1450                                      default=False)
1451     ip_check = BooleanField(verbose_name=_("IP Check"), default=True)
1452     name_check = models.BooleanField(verbose_name=_('DNS Name Check'),
1453                                      default=True)
1454     iallocator = models.BooleanField(verbose_name=_('Automatic Allocation'),
1455                                      default=False)
1456     iallocator_hostname = models.CharField(max_length=255, blank=True)
1457     disk_template = models.CharField(verbose_name=_('Disk Template'),
1458                                      max_length=16)
1459     # XXX why aren't these FKs?
1460     pnode = models.CharField(verbose_name=_('Primary Node'), max_length=255,
1461                              default="")
1462     snode = models.CharField(verbose_name=_('Secondary Node'), max_length=255,
1463                              default="")
1464     os = models.CharField(verbose_name=_('Operating System'), max_length=255)
1465
1466     # Backend parameters (BEPARAMS)
1467     vcpus = models.IntegerField(verbose_name=_('Virtual CPUs'),
1468                                 validators=[MinValueValidator(1)], null=True,
1469                                 blank=True)
1470     # XXX do we really want the minimum memory to be 100MiB? This isn't
1471     # strictly necessary AFAICT.
1472     memory = models.IntegerField(verbose_name=_('Memory'),
1473                                  validators=[MinValueValidator(100)],
1474                                  null=True, blank=True)
1475     minmem = models.IntegerField(verbose_name=_('Minimum Memory'),
1476                                  validators=[MinValueValidator(100)],
1477                                  null=True, blank=True)
1478     disks = PickleField(verbose_name=_('Disks'), null=True, blank=True)
1479     # XXX why isn't this an enum?
1480     disk_type = models.CharField(verbose_name=_('Disk Type'), max_length=255,
1481                                  default="")
1482     nics = PickleField(verbose_name=_('NICs'), null=True, blank=True)
1483     # XXX why isn't this an enum?
1484     nic_type = models.CharField(verbose_name=_('NIC Type'), max_length=255,
1485                                 default="")
1486
1487     # Hypervisor parameters (HVPARAMS)
1488     kernel_path = models.CharField(verbose_name=_('Kernel Path'),
1489                                    max_length=255, default="", blank=True)
1490     root_path = models.CharField(verbose_name=_('Root Path'), max_length=255,
1491                                  default='/', blank=True)
1492     serial_console = models.BooleanField(
1493         verbose_name=_('Enable Serial Console'))
1494     boot_order = models.CharField(verbose_name=_('Boot Device'),
1495                                   max_length=255, default="")
1496     cdrom_image_path = models.CharField(verbose_name=_('CD-ROM Image Path'),
1497                                         max_length=512, blank=True)
1498     cdrom2_image_path = models.CharField(
1499         verbose_name=_('CD-ROM 2 Image Path'),
1500         max_length=512, blank=True)
1501
1502     class Meta:
1503         unique_together = (("cluster", "template_name"),)
1504
1505     def __unicode__(self):
1506         if self.temporary:
1507             return u'(temporary)'
1508         else:
1509             return self.template_name
1510
1511     def set_name(self, name):
1512         """
1513         Set this template's name.
1514
1515         If the name is blank, this template will become temporary and its name
1516         will be set to a unique timestamp.
1517         """
1518
1519         if name:
1520             self.template_name = name
1521         else:
1522             # The template is temporary and will be removed by the VM when the
1523             # VM successfully comes into existence.
1524             self.temporary = True
1525             # Give it a temporary name. Something unique. This is the number
1526             # of microseconds since the epoch; I figure that it'll work out
1527             # alright.
1528             self.template_name = str(int(time.time() * (10 ** 6)))
1529
1530
1531 class GanetiError(models.Model):
1532     """
1533     Class for storing errors which occured in Ganeti
1534     """
1535     cluster = models.ForeignKey(Cluster, related_name="errors")
1536     msg = models.TextField()
1537     code = models.PositiveIntegerField(blank=True, null=True)
1538
1539     # XXX could be fixed with django-model-util's TimeStampedModel
1540     timestamp = models.DateTimeField()
1541
1542     # determines if the errors still appears or not
1543     cleared = models.BooleanField(default=False)
1544
1545     # cluster object (cluster, VM, Node) affected by the error (if any)
1546     obj_type = models.ForeignKey(ContentType, related_name="ganeti_errors")
1547     obj_id = models.PositiveIntegerField()
1548     obj = GenericForeignKey("obj_type", "obj_id")
1549
1550     objects = QuerySetManager()
1551
1552     class Meta:
1553         ordering = ("-timestamp", "code", "msg")
1554
1555     def __unicode__(self):
1556         base = u"[%s] %s" % (self.timestamp, self.msg)
1557         return base
1558
1559     class QuerySet(QuerySet):
1560
1561         def clear_errors(self, obj=None):
1562             """
1563             Clear errors instead of deleting them.
1564             """
1565
1566             qs = self.filter(cleared=False)
1567
1568             if obj:
1569                 qs = qs.get_errors(obj)
1570
1571             return qs.update(cleared=True)
1572
1573         def get_errors(self, obj):
1574             """
1575             Manager method used for getting QuerySet of all errors depending
1576             on passed arguments.
1577
1578             @param  obj   affected object (itself or just QuerySet)
1579             """
1580
1581             if obj is None:
1582                 raise RuntimeError("Implementation error calling get_errors()"
1583                                    "with None")
1584
1585             # Create base query of errors to return.
1586             #
1587             # if it's a Cluster or a queryset for Clusters, then we need to
1588             # get all errors from the Clusters. Do this by filtering on
1589             # GanetiError.cluster instead of obj_id.
1590             if isinstance(obj, (Cluster,)):
1591                 return self.filter(cluster=obj)
1592
1593             elif isinstance(obj, (QuerySet,)):
1594                 if obj.model == Cluster:
1595                     return self.filter(cluster__in=obj)
1596                 else:
1597                     ct = ContentType.objects.get_for_model(obj.model)
1598                     return self.filter(obj_type=ct, obj_id__in=obj)
1599
1600             else:
1601                 ct = ContentType.objects.get_for_model(obj.__class__)
1602                 return self.filter(obj_type=ct, obj_id=obj.pk)
1603
1604     def __repr__(self):
1605         return "<GanetiError '%s'>" % self.msg
1606
1607     @classmethod
1608     def store_error(cls, msg, obj, code, **kwargs):
1609         """
1610         Create and save an error with the given information.
1611
1612         @param  msg  error's message
1613         @param  obj  object (i.e. cluster or vm) affected by the error
1614         @param code  error's code number
1615         """
1616         ct = ContentType.objects.get_for_model(obj.__class__)
1617         is_cluster = isinstance(obj, Cluster)
1618
1619         # 401 -- bad permissions
1620         # 401 is cluster-specific error and thus shouldn't appear on any other
1621         # object.
1622         if code == 401:
1623             if not is_cluster:
1624                 # NOTE: what we do here is almost like:
1625                 #  return self.store_error(msg=msg, code=code, obj=obj.cluster)
1626                 # we just omit the recursiveness
1627                 obj = obj.cluster
1628                 ct = ContentType.objects.get_for_model(Cluster)
1629                 is_cluster = True
1630
1631         # 404 -- object not found
1632         # 404 can occur on any object, but when it occurs on a cluster, then
1633         # any of its children must not see the error again
1634         elif code == 404:
1635             if not is_cluster:
1636                 # return if the error exists for cluster
1637                 try:
1638                     c_ct = ContentType.objects.get_for_model(Cluster)
1639                     return cls.objects.filter(msg=msg, obj_type=c_ct,
1640                                               code=code,
1641                                               obj_id=obj.cluster_id,
1642                                               cleared=False)[0]
1643
1644                 except (cls.DoesNotExist, IndexError):
1645                     # we want to proceed when the error is not
1646                     # cluster-specific
1647                     pass
1648
1649         # XXX use a try/except instead of get_or_create().  get_or_create()
1650         # does not allow us to set cluster_id.  This means we'd have to query
1651         # the cluster object to create the error.  we can't guaranteee the
1652         # cluster will already be queried so use create() instead which does
1653         # allow cluster_id
1654         try:
1655             return cls.objects.filter(msg=msg, obj_type=ct, obj_id=obj.pk,
1656                                       code=code, **kwargs)[0]
1657
1658         except (cls.DoesNotExist, IndexError):
1659             cluster_id = obj.pk if is_cluster else obj.cluster_id
1660
1661             return cls.objects.create(timestamp=datetime.now(), msg=msg,
1662                                       obj_type=ct, obj_id=obj.pk,
1663                                       cluster_id=cluster_id, code=code,
1664                                       **kwargs)
1665
1666
1667 class ClusterUser(models.Model):
1668     """
1669     Base class for objects that may interact with a Cluster or VirtualMachine.
1670     """
1671
1672     name = models.CharField(max_length=128)
1673     real_type = models.ForeignKey(ContentType, related_name="+",
1674                                   editable=False, null=True, blank=True)
1675
1676     def __unicode__(self):
1677         return self.name
1678
1679     def save(self, *args, **kwargs):
1680         if not self.id:
1681             self.real_type = self._get_real_type()
1682         super(ClusterUser, self).save(*args, **kwargs)
1683
1684     def get_absolute_url(self):
1685         return self.cast().get_absolute_url()
1686
1687     @property
1688     def permissable(self):
1689         """ returns an object that can be granted permissions """
1690         return self.cast().permissable
1691
1692     @classmethod
1693     def _get_real_type(cls):
1694         return ContentType.objects.get_for_model(cls)
1695
1696     def cast(self):
1697         return self.real_type.get_object_for_this_type(pk=self.pk)
1698
1699     def used_resources(self, cluster=None, only_running=True):
1700         """
1701         Return dictionary of total resources used by VMs that this ClusterUser
1702         has perms to.
1703         @param cluster  if set, get only VMs from specified cluster
1704         @param only_running  if set, get only running VMs
1705         """
1706         # XXX - order_by must be cleared or it breaks annotation grouping since
1707         #       the default order_by field is also added to the group_by clause
1708         base = self.virtual_machines.all().order_by()
1709
1710         # XXX - use a custom aggregate for ram and vcpu count when filtering by
1711         # running.  this allows us to execute a single query.
1712         #
1713         # XXX - quotes must be used in this order.  postgresql quirk
1714         if only_running:
1715             sum_ram = SumIf('ram', condition="status='running'")
1716             sum_vcpus = SumIf('virtual_cpus', condition="status='running'")
1717         else:
1718             sum_ram = Sum('ram')
1719             sum_vcpus = Sum('virtual_cpus')
1720
1721         base = base.exclude(ram=-1, disk_size=-1, virtual_cpus=-1)
1722
1723         if cluster:
1724             base = base.filter(cluster=cluster)
1725             result = base.aggregate(ram=sum_ram, disk=Sum('disk_size'),
1726                                     virtual_cpus=sum_vcpus)
1727
1728             # repack with zeros instead of Nones
1729             if result['disk'] is None:
1730                 result['disk'] = 0
1731             if result['ram'] is None:
1732                 result['ram'] = 0
1733             if result['virtual_cpus'] is None:
1734                 result['virtual_cpus'] = 0
1735             return result
1736
1737         else:
1738             base = base.values('cluster').annotate(uram=sum_ram,
1739                                                    udisk=Sum('disk_size'),
1740                                                    uvirtual_cpus=sum_vcpus)
1741
1742             # repack as dictionary
1743             result = {}
1744             for used in base:
1745                 # repack with zeros instead of Nones, change index names
1746                 used["ram"] = used.pop("uram") or 0
1747                 used["disk"] = used.pop("udisk") or 0
1748                 used["virtual_cpus"] = used.pop("uvirtual_cpus") or 0
1749                 result[used.pop('cluster')] = used
1750
1751             return result
1752
1753
1754 class Profile(ClusterUser):
1755     """
1756     Profile associated with a django.contrib.auth.User object.
1757     """
1758     user = models.OneToOneField(User)
1759
1760     def get_absolute_url(self):
1761         return self.user.get_absolute_url()
1762
1763     def grant(self, perm, obj):
1764         self.user.grant(perm, obj)
1765
1766     def set_perms(self, perms, obj):
1767         self.user.set_perms(perms, obj)
1768
1769     def get_objects_any_perms(self, *args, **kwargs):
1770         return self.user.get_objects_any_perms(*args, **kwargs)
1771
1772     def has_perm(self, *args, **kwargs):
1773         return self.user.has_perm(*args, **kwargs)
1774
1775     @property
1776     def permissable(self):
1777         """ returns an object that can be granted permissions """
1778         return self.user
1779
1780
1781 class Organization(ClusterUser):
1782     """
1783     An organization is used for grouping Users.
1784
1785     Organizations are matched with an instance of contrib.auth.models.Group.
1786     This model exists so that contrib.auth.models.Group have a 1:1 relation
1787     with a ClusterUser on which quotas and permissions can be assigned.
1788     """
1789
1790     group = models.OneToOneField(Group, related_name='organization')
1791
1792     def get_absolute_url(self):
1793         return self.group.get_absolute_url()
1794
1795     def grant(self, perm, object):
1796         self.group.grant(perm, object)
1797
1798     def set_perms(self, perms, object):
1799         self.group.set_perms(perms, object)
1800
1801     def get_objects_any_perms(self, *args, **kwargs):
1802         return self.group.get_objects_any_perms(*args, **kwargs)
1803
1804     def has_perm(self, *args, **kwargs):
1805         return self.group.has_perm(*args, **kwargs)
1806
1807     @property
1808     def permissable(self):
1809         """ returns an object that can be granted permissions """
1810         return self.group
1811
1812
1813 class Quota(models.Model):
1814     """
1815     A resource limit imposed on a ClusterUser for a given Cluster.  The
1816     attributes of this model represent maximum values the ClusterUser can
1817     consume.  The absence of a Quota indicates unlimited usage.
1818     """
1819     user = models.ForeignKey(ClusterUser, related_name='quotas')
1820     cluster = models.ForeignKey(Cluster, related_name='quotas')
1821
1822     ram = models.IntegerField(default=0, null=True, blank=True)
1823     disk = models.IntegerField(default=0, null=True, blank=True)
1824     virtual_cpus = models.IntegerField(default=0, null=True, blank=True)
1825
1826
1827 class SSHKey(models.Model):
1828     """
1829     Model representing user's SSH public key. Virtual machines rely on
1830     many ssh keys.
1831     """
1832     key = models.TextField(validators=[validate_sshkey])
1833     #filename = models.CharField(max_length=128) # saves key file's name
1834     user = models.ForeignKey(User, related_name='ssh_keys')
1835
1836
1837 def create_profile(sender, instance, **kwargs):
1838     """
1839     Create a profile object whenever a new user is created, also keeps the
1840     profile name synchronized with the username
1841     """
1842     try:
1843         profile, new = Profile.objects.get_or_create(user=instance)
1844         if profile.name != instance.username:
1845             profile.name = instance.username
1846             profile.save()
1847     except DatabaseError:
1848         # XXX - since we're using south to track migrations the Profile table
1849         # won't be available the first time syncdb is run.  Catch the error
1850         # here and let the south migration handle it.
1851         pass
1852
1853
1854 def update_cluster_hash(sender, instance, **kwargs):
1855     """
1856     Updates the Cluster hash for all of it's VirtualMachines, Nodes, and Jobs
1857     """
1858     instance.virtual_machines.all().update(cluster_hash=instance.hash)
1859     instance.jobs.all().update(cluster_hash=instance.hash)
1860     instance.nodes.all().update(cluster_hash=instance.hash)
1861
1862
1863 def update_organization(sender, instance, **kwargs):
1864     """
1865     Creates a Organizations whenever a contrib.auth.models.Group is created
1866     """
1867     org, new = Organization.objects.get_or_create(group=instance)
1868     org.name = instance.name
1869     org.save()
1870
1871 post_save.connect(create_profile, sender=User)
1872 post_save.connect(update_cluster_hash, sender=Cluster)
1873 post_save.connect(update_organization, sender=Group)
1874
1875 # Disconnect create_default_site from django.contrib.sites so that
1876 #  the useless table for sites is not created. This will be
1877 #  reconnected for other apps to use in update_sites_module.
1878 post_syncdb.disconnect(create_default_site, sender=sites_app)
1879 post_syncdb.connect(management.update_sites_module, sender=sites_app,
1880                     dispatch_uid="ganeti.management.update_sites_module")
1881
1882
1883 def regenerate_cu_children(sender, **kwargs):
1884     """
1885     Resets may destroy Profiles and/or Organizations. We need to regenerate
1886     them.
1887     """
1888
1889     # So. What are we actually doing here?
1890     # Whenever a User or Group is saved, the associated Profile or
1891     # Organization is also updated. This means that, if a Profile for a User
1892     # is absent, it will be created.
1893     # More importantly, *why* might a Profile be missing? Simple. Resets of
1894     # the ganeti app destroy them. This shouldn't happen in production, and
1895     # only occasionally in development, but it's good to explicitly handle
1896     # this particular case so that missing Profiles not resulting from a reset
1897     # are easier to diagnose.
1898     try:
1899         for user in User.objects.filter(profile__isnull=True):
1900             user.save()
1901         for group in Group.objects.filter(organization__isnull=True):
1902             group.save()
1903     except DatabaseError:
1904         # XXX - since we're using south to track migrations the Profile table
1905         # won't be available the first time syncdb is run.  Catch the error
1906         # here and let the south migration handle it.
1907         pass
1908
1909 post_syncdb.connect(regenerate_cu_children)
1910
1911
1912 def log_group_create(sender, editor, **kwargs):
1913     """ log group creation signal """
1914     log_action('CREATE', editor, sender)
1915
1916
1917 def log_group_edit(sender, editor, **kwargs):
1918     """ log group edit signal """
1919     log_action('EDIT', editor, sender)
1920
1921
1922 muddle_user_signals.view_group_created.connect(log_group_create)
1923 muddle_user_signals.view_group_edited.connect(log_group_edit)
1924
1925
1926 def refresh_objects(sender, **kwargs):
1927     """
1928     This was originally the code in the 0009
1929     and then 0010 'force_object_refresh' migration
1930
1931     Force a refresh of all Cluster, Nodes, and VirtualMachines, and
1932     import any new Nodes.
1933     """
1934
1935     if kwargs.get('app', False) and kwargs['app'] == 'ganeti_web':
1936         Cluster.objects.all().update(mtime=None)
1937         Node.objects.all().update(mtime=None)
1938         VirtualMachine.objects.all().update(mtime=None)
1939
1940         write = sys.stdout.write
1941         flush = sys.stdout.flush
1942
1943         def wf(str, newline=False):
1944             if newline:
1945                 write('\n')
1946             write(str)
1947             flush()
1948
1949         wf('- Refresh Cached Cluster Objects')
1950         wf(' > Synchronizing Cluster Nodes ', True)
1951         flush()
1952         for cluster in Cluster.objects.all().iterator():
1953             try:
1954                 cluster.sync_nodes()
1955                 wf('.')
1956             except GanetiApiError:
1957                 wf('E')
1958
1959         wf(' > Refreshing Node Caches ', True)
1960         for node in Node.objects.all().iterator():
1961             try:
1962                 wf('.')
1963             except GanetiApiError:
1964                 wf('E')
1965
1966         wf(' > Refreshing Instance Caches ', True)
1967         for instance in VirtualMachine.objects.all().iterator():
1968             try:
1969                 wf('.')
1970             except GanetiApiError:
1971                 wf('E')
1972         wf('\n')
1973
1974
1975 # Set this as post_migrate hook.
1976 post_migrate.connect(refresh_objects)
1977
1978 # Register permissions on our models.
1979 # These are part of the DB schema and should not be changed without serious
1980 # forethought.
1981 # You *must* syncdb after you change these.
1982 register(permissions.CLUSTER_PARAMS, Cluster, 'ganeti_web')
1983 register(permissions.VIRTUAL_MACHINE_PARAMS, VirtualMachine, 'ganeti_web')
1984
1985
1986 # register log actions
1987 register_log_actions()