srw/python.init

   1 import sys
   2 sys.path.append('/tmp/dnet/lib')
   3 sys.path.append('/tmp/test/history.2')
   4
   5 from libelliptics_python import *
   6
   7 # groups used in metadata write
   8 pohmelfs_groups = [1, 2, 3]
   9 pohmelfs_log_file = '/tmp/test/history.2/python.log'
  10
  11 log = elliptics_log_file(pohmelfs_log_file, 10)
  12 n = elliptics_node_python(log)
  13 # we should only add own local group, since we do not want all updates to be repeated for all groups
  14 n.add_groups([2])
  15 n.add_remote('172.16.239.1', 1025)
  16
  17 __return_data = 'unused'
  18
  19 import gc
  20
  21 import struct
  22 from sstable2 import sstable
  23
  24 import logging
  25 FORMAT = "%(asctime)-15s %(process)d %(script)s %(dentry_name)s %(message)s"
  26 logging.basicConfig(filename=pohmelfs_log_file, level=logging.DEBUG, format=FORMAT)
  27
  28 pohmelfs_offset = 0
  29 pohmelfs_size = 0
  30 # do not check csum
  31 #pohmelfs_ioflags_read = 256
  32 pohmelfs_ioflags_read = 0
  33 pohmelfs_ioflags_write = 0
  34 # do not lock operation, since we are 'inside' DNET_CMD_EXEC command already
  35 pohmelfs_aflags = 16
  36 pohmelfs_column = 0
  37 pohmelfs_link_number_column = 2
  38 pohmelfs_inode_info_column = 3
  39 pohmelfs_group_id = 0
  40
  41 def pohmelfs_write(parent_id, content):
  42         n.write_data(parent_id, content, pohmelfs_offset, pohmelfs_aflags, pohmelfs_ioflags_write)
  43         n.write_metadata(parent_id, '', pohmelfs_groups, pohmelfs_aflags)
  44
  45 def parse(buffer):
  46         # Header format:
  47         # 8 bytes: magic
  48         # 2 bytes: version
  49         # 2 bytes: chunk size
  50         # 4 bytes: number of chunks inside
  51         header_fmt = "<8sHHI"
  52
  53         # Chunk header format:
  54         # 2 bytes: length in chunks
  55         # 2 bytes: current chunk
  56         # 2 bytes: key size
  57         # 2 bytes: payload size
  58         chunk_header_fmt = "<HHHH"
  59         chunk_header_size = 8
  60
  61         # Parse header
  62         position = 0
  63         header = struct.unpack_from(header_fmt, buffer, position)
  64         position += struct.calcsize(header_fmt)
  65
  66         header_chunk_size = header[2]
  67         header_count = header[3]
  68         header_strings_start = position
  69
  70         keys = ''
  71         #print "chunk size:", header_chunk_size, ", chunks count:", header_count, ", strings start at", header_strings_start
  72
  73
  74         offset = header_strings_start
  75         while offset < header_strings_start + header_count * header_chunk_size:
  76                 chunk_header = struct.unpack_from(chunk_header_fmt, buffer, offset)
  77                 if chunk_header[1] > 0:
  78                         offset -= chunk_header[1]*header_chunk_size
  79
  80                 chunk_header = struct.unpack_from(chunk_header_fmt, buffer, offset)
  81
  82                 key_len = chunk_header[2]
  83                 payload_len = chunk_header[3]
  84
  85                 key = ""
  86                 payload = ""
  87                 rec_offset = 0
  88
  89                 for i in xrange(0, chunk_header[0]):
  90                         key_size = 0
  91
  92                         # Get key from chunks
  93                         if key_len > 0:
  94                                 key_size = header_chunk_size - chunk_header_size
  95                                 if key_len < key_size:
  96                                         key_size = key_len
  97
  98                                 key_offset = offset + rec_offset + chunk_header_size
  99                                 key += buffer[key_offset:(key_offset + key_size)]
 100                                 key_len -= key_size
 101
 102                         # Get payload from chunks
 103                         if key_len == 0 and payload_len > 0:
 104                                 payload_size = header_chunk_size - chunk_header_size - key_size
 105                                 if payload_len < payload_size:
 106                                         payload_size = payload_len
 107
 108                                 payload_offset = offset + rec_offset + chunk_header_size + key_size
 109                                 payload += buffer[payload_offset:(payload_offset + payload_size)]
 110                                 payload_len -= payload_size
 111
 112                         rec_offset += header_chunk_size
 113                 keys += key + ' '
 114                 #print key, payload
 115                 offset += chunk_header[0]*header_chunk_size
 116
 117         return keys