bindings/python/elliptics.py

   1 #!/usr/bin/python
   2 # -*- coding: utf-8 -*-
   3 import hashlib
   4 import binascii
   5 import struct
   6
   7 import libelliptics_python
   8 from libelliptics_python import log_level, command_flags, io_flags
   9
  10
  11 class NodeStatus(libelliptics_python.dnet_node_status):
  12     def __repr__(self):
  13         return "<NodeStatus nflags:%x, status_flags:%x, log_mask:%x>" % (self.nflags, self.status_flags, self.log_mask)
  14
  15
  16 class Id(libelliptics_python.elliptics_id):
  17     """
  18      Elliptics ID wrapper
  19      It has 2 constructors:
  20          Id()
  21          Id(list_id, group type)
  22
  23      list_id - list of 64 bytes, id of the object itself
  24      group - group ID of the object
  25      type - column
  26     """
  27     pass
  28
  29
  30 class Range(libelliptics_python.elliptics_range):
  31     """
  32      Structure that describes range request
  33      start, end - IDs of the start and the end of the range
  34      offset, size - offset to read from and size of bytes to read, applied for each key
  35      cflags - command flags like locking so on (default is 0, see cflags class for defenitions)
  36      ioflags - command IO flags (default is 0, see ioflags class for definitions)
  37      group - group ID of the object
  38      type - column
  39     """
  40     pass
  41
  42
  43 class Logger(libelliptics_python.elliptics_log_file):
  44     """
  45      Logger, that needed in Node constructor
  46      Constructor takes 2 arguments: log file name (default is "/dev/stderr")
  47      and log level (default is log_level.error, see log_level class for definitions)
  48     """
  49     log_file_name = ""
  50     llevel = log_level.error
  51
  52     def __init__(self, log_file_name = "/dev/stderr", llevel = log_level.error):
  53         """
  54           log_file_name - name of the log file, default value is "/dev/stderr"
  55           log_level - logging level, see log_level class for definitions
  56         """
  57         self.log_file_name = log_file_name
  58         self.llevel = llevel
  59         super(Logger, self).__init__(log_file_name, llevel)
  60
  61     def log(self, message, level):
  62         """
  63           log some message into elliptics log file
  64           message - text message
  65           level - log level, default is log_level.error (see log_level class for definitions)
  66         """
  67         super(Logger, self).log(level, message)
  68
  69     def __repr__(self):
  70         return "<Logger log_file_name:\"%s\" log_level:%d>" % (self.log_file_name, self.llevel)
  71
  72
  73 class Node(libelliptics_python.elliptics_node_python):
  74     """
  75      Main client class. Constructor takes 1 argument: Logger object
  76      """
  77
  78     def __init__(self, log=None):
  79         """
  80           log - Logger object
  81         """
  82         super(Node, self).__init__(log or Logger())
  83
  84     def add_remote(self, addr, port, family=2):
  85         """
  86           Add address of elliptics storage node and connect to it
  87           Usually you do not want to exit if client failed to connect to remote node, so catch up exceptions
  88           But if no remote nodes were successfully added (check get_routes()) then client should not continue
  89           addr - storage address
  90           port - storage port
  91           family - IP protocol family: 2 for IPv4 (default value) and 10 for IPv6
  92         """
  93         super(Node, self).add_remote(addr, port, family)
  94
  95     def add_groups(self, groups):
  96         """
  97           Set groups to work with
  98           groups - list of groups
  99         """
 100         super(Node, self).add_groups(groups)
 101
 102     @property
 103     def groups(self):
 104         return super(Node, self).get_groups()
 105
 106     def get_routes(self):
 107         """
 108           Get routing table
 109
 110           return value:
 111           list - list of node addresses
 112         """
 113         return super(Node, self).get_routes()
 114
 115     def stat_log(self):
 116         """
 117           Get nodes statistics
 118
 119           return value:
 120           string - storage nodes statistics
 121         """
 122         return super(Node, self).stat_log()
 123
 124     def lookup_addr(self, *args, **kwargs):
 125         """
 126           Lookup where key should be located by its ID or key and group_id pair
 127           signatures:
 128               lookup_addr(key, group_id)
 129               lookup_addr(id)
 130
 131           key - remote key name
 132           group_id - group
 133           id - object of Id class
 134
 135           return value:
 136           string - node address
 137         """
 138         return super(Node, self).lookup_addr(*args, **{})
 139
 140     def read_file(self, key, filename, offset = 0, size = 0, column = 0):
 141         """
 142           Read file from elliptics by name/ID
 143           signatures:
 144               read_file(key, filename, offset, size, column)
 145               read_file(id, filename, offset, size)
 146
 147           key - remote key name
 148           column - column type (default is 0, 1 is reserved for metadata)
 149           id - object of Id class
 150
 151           filename - name of local file where data will be saved
 152           offset - read file from this offset (default 0)
 153           size - number of bytes to read, 0 means whole file (default is 0)
 154         """
 155         if isinstance(key, basestring):
 156             new_args = [str(key), filename, offset, size, column]
 157         else:
 158             new_args = [key, filename, offset, size]
 159
 160         super(Node, self).read_file(*new_args)
 161
 162     def write_file(self, key, filename, local_offset = 0, offset = 0, size = 0, \
 163                     cflags = command_flags.default, ioflags = io_flags.default, column = 0):
 164         """
 165           Write file into elliptics by name/ID
 166           signatures:
 167               write_file(key, filename, local_offset, offset, size, cflags, ioflags, column)
 168               write_file(id, filename, local_offset, offset, size, cflags, ioflags)
 169
 170           key - remote key name
 171           column - column type (default is 0, 1 is reserved for metadata)
 172           id - object of Id class
 173
 174           filename - name of local file
 175           local_offset - read local file from this offset (default 0)
 176           offset - write file from this offset (default 0)
 177           size - number of bytes to write, 0 means whole file (default is 0)
 178           cflags - command flags (default is 0, see command_flags class for definitions)
 179           ioflags - command IO flags (default is 0, see io_flags class for definitions)
 180         """
 181         if isinstance(key, basestring):
 182             new_args = [str(key), filename, local_offset, offset, size, cflags, ioflags, column]
 183         else:
 184             new_args = [key, filename, local_offset, offset, size, cflags, ioflags]
 185
 186         super(Node, self).read_file(*new_args)
 187
 188     def _create_read_args(self, key, offset = 0, size = 0, cflags = command_flags.default, ioflags = io_flags.default, column = 0):
 189         if isinstance(key, basestring):
 190             return [str(key), offset, size, cflags, ioflags, column]
 191         else:
 192             return [key, offset, size, cflags, ioflags]
 193
 194     def read_data(self, key, offset = 0, size = 0, cflags = command_flags.default, ioflags = io_flags.default, column = 0):
 195         """
 196           Read data from elliptics by name/ID
 197           signatures:
 198               read_data(key, offset, size, cflags, ioflags, column)
 199               read_data(id, offset, size, cflags, ioflags)
 200
 201           key - remote key name
 202           column - column type (default is 0, 1 is reserved for metadata)
 203           id - object of Id class
 204
 205           offset - read file from this offset (default 0)
 206           size - number of bytes to read, 0 means whole file (default is 0)
 207           cflags - command flags (default is 0, see command_flags class for definitions)
 208           ioflags - command IO flags (default is 0, see io_flags class for definitions)
 209
 210           return value:
 211           string - key value content
 212         """
 213         return super(Node, self).read_data(*self._create_read_args(key, offset, size, cflags, ioflags, column))
 214
 215     read = read_data
 216
 217     def read_latest(self, key, offset = 0, size = 0, cflags = command_flags.default, ioflags = io_flags.default, column = 0):
 218         """
 219           Read data from elliptics by name/ID with the latest update_date in metadata
 220           signatures:
 221               read_latest(key, offset, size, cflags, ioflags, column)
 222               read_latest(id, offset, size, cflags, ioflags)
 223
 224           key - remote key name
 225           column - column type (default is 0, 1 is reserved for metadata)
 226           id - object of Id class
 227
 228           offset - read file from this offset (default 0)
 229           size - number of bytes to read, 0 means whole file (default is 0)
 230           cflags - command flags flags (default is 0, see command_flags class for definitions)
 231           ioflags - command IO flags (default is 0, see io_flags class for definitions)
 232
 233           return value:
 234           string - key value content
 235         """
 236         return super(Node, self).read_latest(*self._create_read_args(key, offset, size, cflags, ioflags, column))
 237
 238     def create_write_args(self, key, data, offset, ioflags, cflags, column):
 239         if isinstance(key, basestring):
 240             return [str(key), data, offset, cflags, ioflags, column]
 241         else:
 242             return [key, data, offset, cflags, ioflags]
 243
 244     def write_data(self, key, data, offset = 0, cflags = command_flags.default, ioflags = io_flags.default, column = 0):
 245         """
 246          Write data into elliptics by name/ID
 247          signatures:
 248              write_data(key, data, offset, cflags, ioflags, column)
 249              write_data(id, data, offset, cflags, ioflags)
 250
 251          key - remote key name
 252          column - column type (default is 0, 1 is reserved for metadata)
 253          id - object of Id class
 254
 255          data - data to be written
 256          offset - write data in remote from this offset (default 0)
 257          cflags - command flags flags (default is 0, see command_flags class for definitions)
 258          ioflags - command IO flags (default is 0, see io_flags class for definitions)
 259
 260          return value:
 261          string - nodes and paths where data was stored
 262          """
 263         return super(Node, self).write_data(*self.create_write_args(key, data, offset, ioflags, cflags, column))
 264
 265     def write_metadata(self, key, cflags = command_flags.default, name = None, groups = None):
 266         """
 267          Write metadata into elliptics by name/ID
 268          signatures:
 269              write_metadata(key, cflags)
 270              write_metadata(id, name, groups, cflags)
 271
 272          key - remote key name
 273          id - object of Id class
 274
 275          name - key name
 276          groups - groups where data was stored
 277          cflags - command flags (default is 0, see command_flags class for definitions)
 278         """
 279         if isinstance(key, basestring):
 280             new_args = [str(key), cflags]
 281         else:
 282             new_args = [key, name, groups, cflags]
 283
 284         super(Node, self).write_metadata(*new_args)
 285
 286     def write(self, key, data):
 287         """
 288         Simple write
 289         """
 290         self.write_data(key, data)
 291         self.write_metadata(key)
 292
 293     def remove(self, key, cflags = command_flags.default, ioflags = io_flags.default, column = 0):
 294         """
 295              Remove key by name/ID
 296              signatures:
 297                  remove(key, cflags, ioflags, column)
 298                  remove(id, cflags, ioflags)
 299
 300              key - remote key name
 301              column - column type (default is 0, 1 is reserved for metadata)
 302              id - object of Id class
 303
 304              cflags - command flags flags (default is 0, see command_flags class for definitions)
 305              ioflags - command IO flags (default is 0, see io_flags class for definitions)
 306         """
 307         if isinstance(key, basestring):
 308             new_args = [str(key), cflags, ioflags, column]
 309         else:
 310             new_args = [key, cflags, ioflags]
 311
 312         super(Node, self).remove(*new_args)
 313
 314     def execute(self, *args, **kwargs):
 315         """
 316              Execite server-side script
 317              signatures:
 318                  exec(id, event, data, binary)
 319                  exec(key, event, data, binary)
 320                  exec(event, data, binary)
 321
 322              key - remote key name
 323              id - object of Id class
 324
 325              event - server-side event name
 326              data - data for given event
 327              binary - binary data (its logical meaning is the same as for data, it was added for convenience)
 328
 329              If execute() is called with 3 arguments script will be started on all storage nodes.
 330              If id or key is specified script will be started on the node which hosts given key/id.
 331
 332              return value:
 333              string - result of the script execution
 334         """
 335         return super(Node, self).execute(*args, **{})
 336
 337     def update_status(self, key, status = None, update = 0):
 338         """
 339              Update elliptics status and log mask
 340              signatures:
 341                  update_status(id, status, update)
 342                  update_status((addr, port, family), status, update)
 343
 344              key - remote key name
 345              id - object of Id class
 346
 347              addr - storage address
 348              port - storage port
 349              family - IP protocol family: 2 for IPv4 (default value) and 10 for IPv6
 350              status - new node status, object of NodeStatus class
 351              update - update status or just return current (default is 0)
 352
 353              If update = 0 status will not be changed
 354
 355              return value:
 356              NodeStatus - current node status
 357         """
 358         status = status or NodeStatus()
 359         if hasattr(key, '__iter__'):
 360             new_args = (key[0], key[1], key[2], status, update)
 361         else:
 362             new_args = (key, status, update)
 363
 364         ret = super(Node, self).update_status(*new_args)
 365         ret.__class__ = NodeStatus
 366         return ret
 367
 368     def bulk_read(self, keys, cflags = command_flags.default, raw=False):
 369         """
 370          Bulk read keys from elliptics
 371          keys - list of keys by name
 372          cflags - command flags (default is 0, see command_flags class for definitions)
 373
 374          return value:
 375          dict: key - original key, value - data itself
 376          if raw is True: list - list of strings, each string consists of 64 byte key (sha-512 of original key), 8 byte data length and data itself
 377         """
 378         if type(keys) in set([tuple, list, set, dict]):
 379             keys = list(keys)
 380
 381         rv = super(Node, self).bulk_read(keys, cflags)
 382
 383         if raw:
 384             return rv
 385
 386         if not rv:
 387             return {}
 388
 389         keys = dict([(hashlib.sha512(key).hexdigest(), key) for key in keys])
 390
 391         rv_dict = {}
 392         for r in rv:
 393             key = binascii.hexlify(r[:64])
 394             data_len = struct.unpack('Q', r[64:72])[0]
 395             data = struct.unpack("%ss" % data_len, r[72:72 + data_len])[0]
 396             rv_dict[keys[key]] = data
 397         return rv_dict
 398
 399
 400
 401     def read_data_range(self, read_range):
 402         """
 403              Read keys from elliptics by range of IDs
 404              read_range - object of Range class
 405
 406              return value:
 407              list - list of strings, each string consists of 64 byte key, 8 byte data length and data itself
 408         """
 409         return super(Node, self).read_data_range(read_range)
 410
 411