new file: pixi.toml
[GalaxyCodeBases.git] / etc / Windows / py-kms / structure.py
blobe1f634fdd5742aee71a8fc85edd23d8548294326
1 # Copyright (c) 2003-2012 CORE Security Technologies
3 # This software is provided under under a slightly modified version
4 # of the Apache Software License. See the accompanying LICENSE file
5 # for more information.
7 # $Id$
10 from struct import pack, unpack, calcsize
12 class Structure:
13 """ sublcasses can define commonHdr and/or structure.
14 each of them is an tuple of either two: (fieldName, format) or three: (fieldName, ':', class) fields.
15 [it can't be a dictionary, because order is important]
17 where format specifies how the data in the field will be converted to/from bytes (string)
18 class is the class to use when unpacking ':' fields.
20 each field can only contain one value (or an array of values for *)
21 i.e. struct.pack('Hl',1,2) is valid, but format specifier 'Hl' is not (you must use 2 dfferent fields)
23 format specifiers:
24 specifiers from module pack can be used with the same format
25 see struct.__doc__ (pack/unpack is finally called)
26 x [padding byte]
27 c [character]
28 b [signed byte]
29 B [unsigned byte]
30 h [signed short]
31 H [unsigned short]
32 l [signed long]
33 L [unsigned long]
34 i [signed integer]
35 I [unsigned integer]
36 q [signed long long (quad)]
37 Q [unsigned long long (quad)]
38 s [string (array of chars), must be preceded with length in format specifier, padded with zeros]
39 p [pascal string (includes byte count), must be preceded with length in format specifier, padded with zeros]
40 f [float]
41 d [double]
42 = [native byte ordering, size and alignment]
43 @ [native byte ordering, standard size and alignment]
44 ! [network byte ordering]
45 < [little endian]
46 > [big endian]
48 usual printf like specifiers can be used (if started with %)
49 [not recommeneded, there is no why to unpack this]
51 %08x will output an 8 bytes hex
52 %s will output a string
53 %s\\x00 will output a NUL terminated string
54 %d%d will output 2 decimal digits (against the very same specification of Structure)
55 ...
57 some additional format specifiers:
58 : just copy the bytes from the field into the output string (input may be string, other structure, or anything responding to __str__()) (for unpacking, all what's left is returned)
59 z same as :, but adds a NUL byte at the end (asciiz) (for unpacking the first NUL byte is used as terminator) [asciiz string]
60 u same as z, but adds two NUL bytes at the end (after padding to an even size with NULs). (same for unpacking) [unicode string]
61 w DCE-RPC/NDR string (it's a macro for [ '<L=(len(field)+1)/2','"\\x00\\x00\\x00\\x00','<L=(len(field)+1)/2',':' ]
62 ?-field length of field named 'field', formated as specified with ? ('?' may be '!H' for example). The input value overrides the real length
63 ?1*?2 array of elements. Each formated as '?2', the number of elements in the array is stored as specified by '?1' (?1 is optional, or can also be a constant (number), for unpacking)
64 'xxxx literal xxxx (field's value doesn't change the output. quotes must not be closed or escaped)
65 "xxxx literal xxxx (field's value doesn't change the output. quotes must not be closed or escaped)
66 _ will not pack the field. Accepts a third argument, which is an unpack code. See _Test_UnpackCode for an example
67 ?=packcode will evaluate packcode in the context of the structure, and pack the result as specified by ?. Unpacking is made plain
68 ?&fieldname "Address of field fieldname".
69 For packing it will simply pack the id() of fieldname. Or use 0 if fieldname doesn't exists.
70 For unpacking, it's used to know weather fieldname has to be unpacked or not, i.e. by adding a & field you turn another field (fieldname) in an optional field.
72 """
73 commonHdr = ()
74 structure = ()
75 debug = 0
77 def __init__(self, data = None, alignment = 0):
78 if not hasattr(self, 'alignment'):
79 self.alignment = alignment
81 self.fields = {}
82 self.rawData = data
83 if data is not None:
84 self.fromString(data)
85 else:
86 self.data = None
88 @classmethod
89 def fromFile(self, file):
90 answer = self()
91 answer.fromString(file.read(len(answer)))
92 return answer
94 def setAlignment(self, alignment):
95 self.alignment = alignment
97 def setData(self, data):
98 self.data = data
100 def packField(self, fieldName, format = None):
101 if self.debug:
102 print "packField( %s | %s )" % (fieldName, format)
104 if format is None:
105 format = self.formatForField(fieldName)
107 if self.fields.has_key(fieldName):
108 ans = self.pack(format, self.fields[fieldName], field = fieldName)
109 else:
110 ans = self.pack(format, None, field = fieldName)
112 if self.debug:
113 print "\tanswer %r" % ans
115 return ans
117 def getData(self):
118 if self.data is not None:
119 return self.data
120 data = ''
121 for field in self.commonHdr+self.structure:
122 try:
123 data += self.packField(field[0], field[1])
124 except Exception, e:
125 if self.fields.has_key(field[0]):
126 e.args += ("When packing field '%s | %s | %r' in %s" % (field[0], field[1], self[field[0]], self.__class__),)
127 else:
128 e.args += ("When packing field '%s | %s' in %s" % (field[0], field[1], self.__class__),)
129 raise
130 if self.alignment:
131 if len(data) % self.alignment:
132 data += ('\x00'*self.alignment)[:-(len(data) % self.alignment)]
134 #if len(data) % self.alignment: data += ('\x00'*self.alignment)[:-(len(data) % self.alignment)]
135 return data
137 def fromString(self, data):
138 self.rawData = data
139 for field in self.commonHdr+self.structure:
140 if self.debug:
141 print "fromString( %s | %s | %r )" % (field[0], field[1], data)
142 size = self.calcUnpackSize(field[1], data, field[0])
143 if self.debug:
144 print " size = %d" % size
145 dataClassOrCode = str
146 if len(field) > 2:
147 dataClassOrCode = field[2]
148 try:
149 self[field[0]] = self.unpack(field[1], data[:size], dataClassOrCode = dataClassOrCode, field = field[0])
150 except Exception,e:
151 e.args += ("When unpacking field '%s | %s | %r[:%d]'" % (field[0], field[1], data, size),)
152 raise
154 size = self.calcPackSize(field[1], self[field[0]], field[0])
155 if self.alignment and size % self.alignment:
156 size += self.alignment - (size % self.alignment)
157 data = data[size:]
159 return self
161 def __setitem__(self, key, value):
162 self.fields[key] = value
163 self.data = None # force recompute
165 def __getitem__(self, key):
166 return self.fields[key]
168 def __delitem__(self, key):
169 del self.fields[key]
171 def __str__(self):
172 return self.getData()
174 def __len__(self):
175 # XXX: improve
176 return len(self.getData())
178 def pack(self, format, data, field = None):
179 if self.debug:
180 print " pack( %s | %r | %s)" % (format, data, field)
182 if field:
183 addressField = self.findAddressFieldFor(field)
184 if (addressField is not None) and (data is None):
185 return ''
187 # void specifier
188 if format[:1] == '_':
189 return ''
191 # quote specifier
192 if format[:1] == "'" or format[:1] == '"':
193 return format[1:]
195 # code specifier
196 two = format.split('=')
197 if len(two) >= 2:
198 try:
199 return self.pack(two[0], data)
200 except:
201 fields = {'self':self}
202 fields.update(self.fields)
203 return self.pack(two[0], eval(two[1], {}, fields))
205 # address specifier
206 two = format.split('&')
207 if len(two) == 2:
208 try:
209 return self.pack(two[0], data)
210 except:
211 if (self.fields.has_key(two[1])) and (self[two[1]] is not None):
212 return self.pack(two[0], id(self[two[1]]) & ((1<<(calcsize(two[0])*8))-1) )
213 else:
214 return self.pack(two[0], 0)
216 # length specifier
217 two = format.split('-')
218 if len(two) == 2:
219 try:
220 return self.pack(two[0],data)
221 except:
222 return self.pack(two[0], self.calcPackFieldSize(two[1]))
224 # array specifier
225 two = format.split('*')
226 if len(two) == 2:
227 answer = ''
228 for each in data:
229 answer += self.pack(two[1], each)
230 if two[0]:
231 if two[0].isdigit():
232 if int(two[0]) != len(data):
233 raise Exception, "Array field has a constant size, and it doesn't match the actual value"
234 else:
235 return self.pack(two[0], len(data))+answer
236 return answer
238 # "printf" string specifier
239 if format[:1] == '%':
240 # format string like specifier
241 return format % data
243 # asciiz specifier
244 if format[:1] == 'z':
245 return str(data)+'\0'
247 # unicode specifier
248 if format[:1] == 'u':
249 return str(data)+'\0\0' + (len(data) & 1 and '\0' or '')
251 # DCE-RPC/NDR string specifier
252 if format[:1] == 'w':
253 if len(data) == 0:
254 data = '\0\0'
255 elif len(data) % 2:
256 data += '\0'
257 l = pack('<L', len(data)/2)
258 return '%s\0\0\0\0%s%s' % (l,l,data)
260 if data is None:
261 raise Exception, "Trying to pack None"
263 # literal specifier
264 if format[:1] == ':':
265 return str(data)
267 # struct like specifier
268 return pack(format, data)
270 def unpack(self, format, data, dataClassOrCode = str, field = None):
271 if self.debug:
272 print " unpack( %s | %r )" % (format, data)
274 if field:
275 addressField = self.findAddressFieldFor(field)
276 if addressField is not None:
277 if not self[addressField]:
278 return
280 # void specifier
281 if format[:1] == '_':
282 if dataClassOrCode != str:
283 fields = {'self':self, 'inputDataLeft':data}
284 fields.update(self.fields)
285 return eval(dataClassOrCode, {}, fields)
286 else:
287 return None
289 # quote specifier
290 if format[:1] == "'" or format[:1] == '"':
291 answer = format[1:]
292 if answer != data:
293 raise Exception, "Unpacked data doesn't match constant value '%r' should be '%r'" % (data, answer)
294 return answer
296 # address specifier
297 two = format.split('&')
298 if len(two) == 2:
299 return self.unpack(two[0],data)
301 # code specifier
302 two = format.split('=')
303 if len(two) >= 2:
304 return self.unpack(two[0],data)
306 # length specifier
307 two = format.split('-')
308 if len(two) == 2:
309 return self.unpack(two[0],data)
311 # array specifier
312 two = format.split('*')
313 if len(two) == 2:
314 answer = []
315 sofar = 0
316 if two[0].isdigit():
317 number = int(two[0])
318 elif two[0]:
319 sofar += self.calcUnpackSize(two[0], data)
320 number = self.unpack(two[0], data[:sofar])
321 else:
322 number = -1
324 while number and sofar < len(data):
325 nsofar = sofar + self.calcUnpackSize(two[1],data[sofar:])
326 answer.append(self.unpack(two[1], data[sofar:nsofar], dataClassOrCode))
327 number -= 1
328 sofar = nsofar
329 return answer
331 # "printf" string specifier
332 if format[:1] == '%':
333 # format string like specifier
334 return format % data
336 # asciiz specifier
337 if format == 'z':
338 if data[-1] != '\x00':
339 raise Exception, ("%s 'z' field is not NUL terminated: %r" % (field, data))
340 return data[:-1] # remove trailing NUL
342 # unicode specifier
343 if format == 'u':
344 if data[-2:] != '\x00\x00':
345 raise Exception, ("%s 'u' field is not NUL-NUL terminated: %r" % (field, data))
346 return data[:-2] # remove trailing NUL
348 # DCE-RPC/NDR string specifier
349 if format == 'w':
350 l = unpack('<L', data[:4])[0]
351 return data[12:12+l*2]
353 # literal specifier
354 if format == ':':
355 return dataClassOrCode(data)
357 # struct like specifier
358 return unpack(format, data)[0]
360 def calcPackSize(self, format, data, field = None):
361 # # print " calcPackSize %s:%r" % (format, data)
362 if field:
363 addressField = self.findAddressFieldFor(field)
364 if addressField is not None:
365 if not self[addressField]:
366 return 0
368 # void specifier
369 if format[:1] == '_':
370 return 0
372 # quote specifier
373 if format[:1] == "'" or format[:1] == '"':
374 return len(format)-1
376 # address specifier
377 two = format.split('&')
378 if len(two) == 2:
379 return self.calcPackSize(two[0], data)
381 # code specifier
382 two = format.split('=')
383 if len(two) >= 2:
384 return self.calcPackSize(two[0], data)
386 # length specifier
387 two = format.split('-')
388 if len(two) == 2:
389 return self.calcPackSize(two[0], data)
391 # array specifier
392 two = format.split('*')
393 if len(two) == 2:
394 answer = 0
395 if two[0].isdigit():
396 if int(two[0]) != len(data):
397 raise Exception, "Array field has a constant size, and it doesn't match the actual value"
398 elif two[0]:
399 answer += self.calcPackSize(two[0], len(data))
401 for each in data:
402 answer += self.calcPackSize(two[1], each)
403 return answer
405 # "printf" string specifier
406 if format[:1] == '%':
407 # format string like specifier
408 return len(format % data)
410 # asciiz specifier
411 if format[:1] == 'z':
412 return len(data)+1
414 # asciiz specifier
415 if format[:1] == 'u':
416 l = len(data)
417 return l + (l & 1 and 3 or 2)
419 # DCE-RPC/NDR string specifier
420 if format[:1] == 'w':
421 l = len(data)
422 return 12+l+l % 2
424 # literal specifier
425 if format[:1] == ':':
426 return len(data)
428 # struct like specifier
429 return calcsize(format)
431 def calcUnpackSize(self, format, data, field = None):
432 if self.debug:
433 print " calcUnpackSize( %s | %s | %r)" % (field, format, data)
435 # void specifier
436 if format[:1] == '_':
437 return 0
439 addressField = self.findAddressFieldFor(field)
440 if addressField is not None:
441 if not self[addressField]:
442 return 0
444 try:
445 lengthField = self.findLengthFieldFor(field)
446 return self[lengthField]
447 except:
448 pass
450 # XXX: Try to match to actual values, raise if no match
452 # quote specifier
453 if format[:1] == "'" or format[:1] == '"':
454 return len(format)-1
456 # address specifier
457 two = format.split('&')
458 if len(two) == 2:
459 return self.calcUnpackSize(two[0], data)
461 # code specifier
462 two = format.split('=')
463 if len(two) >= 2:
464 return self.calcUnpackSize(two[0], data)
466 # length specifier
467 two = format.split('-')
468 if len(two) == 2:
469 return self.calcUnpackSize(two[0], data)
471 # array specifier
472 two = format.split('*')
473 if len(two) == 2:
474 answer = 0
475 if two[0]:
476 if two[0].isdigit():
477 number = int(two[0])
478 else:
479 answer += self.calcUnpackSize(two[0], data)
480 number = self.unpack(two[0], data[:answer])
482 while number:
483 number -= 1
484 answer += self.calcUnpackSize(two[1], data[answer:])
485 else:
486 while answer < len(data):
487 answer += self.calcUnpackSize(two[1], data[answer:])
488 return answer
490 # "printf" string specifier
491 if format[:1] == '%':
492 raise Exception, "Can't guess the size of a printf like specifier for unpacking"
494 # asciiz specifier
495 if format[:1] == 'z':
496 return data.index('\x00')+1
498 # asciiz specifier
499 if format[:1] == 'u':
500 l = data.index('\x00\x00')
501 return l + (l & 1 and 3 or 2)
503 # DCE-RPC/NDR string specifier
504 if format[:1] == 'w':
505 l = unpack('<L', data[:4])[0]
506 return 12+l*2
508 # literal specifier
509 if format[:1] == ':':
510 return len(data)
512 # struct like specifier
513 return calcsize(format)
515 def calcPackFieldSize(self, fieldName, format = None):
516 if format is None:
517 format = self.formatForField(fieldName)
519 return self.calcPackSize(format, self[fieldName])
521 def formatForField(self, fieldName):
522 for field in self.commonHdr+self.structure:
523 if field[0] == fieldName:
524 return field[1]
525 raise Exception, ("Field %s not found" % fieldName)
527 def findAddressFieldFor(self, fieldName):
528 descriptor = '&%s' % fieldName
529 l = len(descriptor)
530 for field in self.commonHdr+self.structure:
531 if field[1][-l:] == descriptor:
532 return field[0]
533 return None
535 def findLengthFieldFor(self, fieldName):
536 descriptor = '-%s' % fieldName
537 l = len(descriptor)
538 for field in self.commonHdr+self.structure:
539 if field[1][-l:] == descriptor:
540 return field[0]
541 return None
543 def zeroValue(self, format):
544 two = format.split('*')
545 if len(two) == 2:
546 if two[0].isdigit():
547 return (self.zeroValue(two[1]),)*int(two[0])
549 if not format.find('*') == -1: return ()
550 if 's' in format: return ''
551 if format in ['z',':','u']: return ''
552 if format == 'w': return '\x00\x00'
554 return 0
556 def clear(self):
557 for field in self.commonHdr + self.structure:
558 self[field[0]] = self.zeroValue(field[1])
560 def dump(self, msg = None, indent = 0):
561 import types
562 if msg is None: msg = self.__class__.__name__
563 ind = ' '*indent
564 print "\n%s" % (msg)
565 fixedFields = []
566 for field in self.commonHdr+self.structure:
567 i = field[0]
568 if i in self.fields:
569 fixedFields.append(i)
570 if isinstance(self[i], Structure):
571 self[i].dump('%s%s:{' % (ind,i), indent = indent + 4)
572 print "%s}" % ind
573 else:
574 print "%s%s: {%r}" % (ind,i,self[i])
575 # Do we have remaining fields not defined in the structures? let's
576 # print them
577 remainingFields = list(set(self.fields) - set(fixedFields))
578 for i in remainingFields:
579 if isinstance(self[i], Structure):
580 self[i].dump('%s%s:{' % (ind,i), indent = indent + 4)
581 print "%s}" % ind
582 else:
583 print "%s%s: {%r}" % (ind,i,self[i])
586 class _StructureTest:
587 alignment = 0
588 def create(self,data = None):
589 if data is not None:
590 return self.theClass(data, alignment = self.alignment)
591 else:
592 return self.theClass(alignment = self.alignment)
594 def run(self):
595 print
596 print "-"*70
597 testName = self.__class__.__name__
598 print "starting test: %s....." % testName
599 a = self.create()
600 self.populate(a)
601 a.dump("packing.....")
602 a_str = str(a)
603 print "packed: %r" % a_str
604 print "unpacking....."
605 b = self.create(a_str)
606 b.dump("unpacked.....")
607 print "repacking....."
608 b_str = str(b)
609 if b_str != a_str:
610 print "ERROR: original packed and repacked don't match"
611 print "packed: %r" % b_str
613 class _Test_simple(_StructureTest):
614 class theClass(Structure):
615 commonHdr = ()
616 structure = (
617 ('int1', '!L'),
618 ('len1','!L-z1'),
619 ('arr1','B*<L'),
620 ('z1', 'z'),
621 ('u1','u'),
622 ('', '"COCA'),
623 ('len2','!H-:1'),
624 ('', '"COCA'),
625 (':1', ':'),
626 ('int3','>L'),
627 ('code1','>L=len(arr1)*2+0x1000'),
630 def populate(self, a):
631 a['default'] = 'hola'
632 a['int1'] = 0x3131
633 a['int3'] = 0x45444342
634 a['z1'] = 'hola'
635 a['u1'] = 'hola'.encode('utf_16_le')
636 a[':1'] = ':1234:'
637 a['arr1'] = (0x12341234,0x88990077,0x41414141)
638 # a['len1'] = 0x42424242
640 class _Test_fixedLength(_Test_simple):
641 def populate(self, a):
642 _Test_simple.populate(self, a)
643 a['len1'] = 0x42424242
645 class _Test_simple_aligned4(_Test_simple):
646 alignment = 4
648 class _Test_nested(_StructureTest):
649 class theClass(Structure):
650 class _Inner(Structure):
651 structure = (('data', 'z'),)
653 structure = (
654 ('nest1', ':', _Inner),
655 ('nest2', ':', _Inner),
656 ('int', '<L'),
659 def populate(self, a):
660 a['nest1'] = _Test_nested.theClass._Inner()
661 a['nest2'] = _Test_nested.theClass._Inner()
662 a['nest1']['data'] = 'hola manola'
663 a['nest2']['data'] = 'chau loco'
664 a['int'] = 0x12345678
666 class _Test_Optional(_StructureTest):
667 class theClass(Structure):
668 structure = (
669 ('pName','<L&Name'),
670 ('pList','<L&List'),
671 ('Name','w'),
672 ('List','<H*<L'),
675 def populate(self, a):
676 a['Name'] = 'Optional test'
677 a['List'] = (1,2,3,4)
679 class _Test_Optional_sparse(_Test_Optional):
680 def populate(self, a):
681 _Test_Optional.populate(self, a)
682 del a['Name']
684 class _Test_AsciiZArray(_StructureTest):
685 class theClass(Structure):
686 structure = (
687 ('head','<L'),
688 ('array','B*z'),
689 ('tail','<L'),
692 def populate(self, a):
693 a['head'] = 0x1234
694 a['tail'] = 0xabcd
695 a['array'] = ('hola','manola','te traje')
697 class _Test_UnpackCode(_StructureTest):
698 class theClass(Structure):
699 structure = (
700 ('leni','<L=len(uno)*2'),
701 ('cuchi','_-uno','leni/2'),
702 ('uno',':'),
703 ('dos',':'),
706 def populate(self, a):
707 a['uno'] = 'soy un loco!'
708 a['dos'] = 'que haces fiera'
710 class _Test_AAA(_StructureTest):
711 class theClass(Structure):
712 commonHdr = ()
713 structure = (
714 ('iv', '!L=((init_vector & 0xFFFFFF) << 8) | ((pad & 0x3f) << 2) | (keyid & 3)'),
715 ('init_vector', '_','(iv >> 8)'),
716 ('pad', '_','((iv >>2) & 0x3F)'),
717 ('keyid', '_','( iv & 0x03 )'),
718 ('dataLen', '_-data', 'len(inputDataLeft)-4'),
719 ('data',':'),
720 ('icv','>L'),
723 def populate(self, a):
724 a['init_vector']=0x01020304
725 #a['pad']=int('01010101',2)
726 a['pad']=int('010101',2)
727 a['keyid']=0x07
728 a['data']="\xA0\xA1\xA2\xA3\xA4\xA5\xA6\xA7\xA8\xA9"
729 a['icv'] = 0x05060708
730 #a['iv'] = 0x01020304
732 if __name__ == '__main__':
733 _Test_simple().run()
735 try:
736 _Test_fixedLength().run()
737 except:
738 print "cannot repack because length is bogus"
740 _Test_simple_aligned4().run()
741 _Test_nested().run()
742 _Test_Optional().run()
743 _Test_Optional_sparse().run()
744 _Test_AsciiZArray().run()
745 _Test_UnpackCode().run()
746 _Test_AAA().run()