1 # Copyright (c) 2003-2012 CORE Security Technologies
3 # This software is provided under under a slightly modified version
4 # of the Apache Software License. See the accompanying LICENSE file
5 # for more information.
10 from struct
import pack
, unpack
, calcsize
13 """ sublcasses can define commonHdr and/or structure.
14 each of them is an tuple of either two: (fieldName, format) or three: (fieldName, ':', class) fields.
15 [it can't be a dictionary, because order is important]
17 where format specifies how the data in the field will be converted to/from bytes (string)
18 class is the class to use when unpacking ':' fields.
20 each field can only contain one value (or an array of values for *)
21 i.e. struct.pack('Hl',1,2) is valid, but format specifier 'Hl' is not (you must use 2 dfferent fields)
24 specifiers from module pack can be used with the same format
25 see struct.__doc__ (pack/unpack is finally called)
36 q [signed long long (quad)]
37 Q [unsigned long long (quad)]
38 s [string (array of chars), must be preceded with length in format specifier, padded with zeros]
39 p [pascal string (includes byte count), must be preceded with length in format specifier, padded with zeros]
42 = [native byte ordering, size and alignment]
43 @ [native byte ordering, standard size and alignment]
44 ! [network byte ordering]
48 usual printf like specifiers can be used (if started with %)
49 [not recommeneded, there is no why to unpack this]
51 %08x will output an 8 bytes hex
52 %s will output a string
53 %s\\x00 will output a NUL terminated string
54 %d%d will output 2 decimal digits (against the very same specification of Structure)
57 some additional format specifiers:
58 : just copy the bytes from the field into the output string (input may be string, other structure, or anything responding to __str__()) (for unpacking, all what's left is returned)
59 z same as :, but adds a NUL byte at the end (asciiz) (for unpacking the first NUL byte is used as terminator) [asciiz string]
60 u same as z, but adds two NUL bytes at the end (after padding to an even size with NULs). (same for unpacking) [unicode string]
61 w DCE-RPC/NDR string (it's a macro for [ '<L=(len(field)+1)/2','"\\x00\\x00\\x00\\x00','<L=(len(field)+1)/2',':' ]
62 ?-field length of field named 'field', formated as specified with ? ('?' may be '!H' for example). The input value overrides the real length
63 ?1*?2 array of elements. Each formated as '?2', the number of elements in the array is stored as specified by '?1' (?1 is optional, or can also be a constant (number), for unpacking)
64 'xxxx literal xxxx (field's value doesn't change the output. quotes must not be closed or escaped)
65 "xxxx literal xxxx (field's value doesn't change the output. quotes must not be closed or escaped)
66 _ will not pack the field. Accepts a third argument, which is an unpack code. See _Test_UnpackCode for an example
67 ?=packcode will evaluate packcode in the context of the structure, and pack the result as specified by ?. Unpacking is made plain
68 ?&fieldname "Address of field fieldname".
69 For packing it will simply pack the id() of fieldname. Or use 0 if fieldname doesn't exists.
70 For unpacking, it's used to know weather fieldname has to be unpacked or not, i.e. by adding a & field you turn another field (fieldname) in an optional field.
77 def __init__(self
, data
= None, alignment
= 0):
78 if not hasattr(self
, 'alignment'):
79 self
.alignment
= alignment
89 def fromFile(self
, file):
91 answer
.fromString(file.read(len(answer
)))
94 def setAlignment(self
, alignment
):
95 self
.alignment
= alignment
97 def setData(self
, data
):
100 def packField(self
, fieldName
, format
= None):
102 print "packField( %s | %s )" % (fieldName
, format
)
105 format
= self
.formatForField(fieldName
)
107 if self
.fields
.has_key(fieldName
):
108 ans
= self
.pack(format
, self
.fields
[fieldName
], field
= fieldName
)
110 ans
= self
.pack(format
, None, field
= fieldName
)
113 print "\tanswer %r" % ans
118 if self
.data
is not None:
121 for field
in self
.commonHdr
+self
.structure
:
123 data
+= self
.packField(field
[0], field
[1])
125 if self
.fields
.has_key(field
[0]):
126 e
.args
+= ("When packing field '%s | %s | %r' in %s" % (field
[0], field
[1], self
[field
[0]], self
.__class
__),)
128 e
.args
+= ("When packing field '%s | %s' in %s" % (field
[0], field
[1], self
.__class
__),)
131 if len(data
) % self
.alignment
:
132 data
+= ('\x00'*self
.alignment
)[:-(len(data
) % self
.alignment
)]
134 #if len(data) % self.alignment: data += ('\x00'*self.alignment)[:-(len(data) % self.alignment)]
137 def fromString(self
, data
):
139 for field
in self
.commonHdr
+self
.structure
:
141 print "fromString( %s | %s | %r )" % (field
[0], field
[1], data
)
142 size
= self
.calcUnpackSize(field
[1], data
, field
[0])
144 print " size = %d" % size
145 dataClassOrCode
= str
147 dataClassOrCode
= field
[2]
149 self
[field
[0]] = self
.unpack(field
[1], data
[:size
], dataClassOrCode
= dataClassOrCode
, field
= field
[0])
151 e
.args
+= ("When unpacking field '%s | %s | %r[:%d]'" % (field
[0], field
[1], data
, size
),)
154 size
= self
.calcPackSize(field
[1], self
[field
[0]], field
[0])
155 if self
.alignment
and size
% self
.alignment
:
156 size
+= self
.alignment
- (size
% self
.alignment
)
161 def __setitem__(self
, key
, value
):
162 self
.fields
[key
] = value
163 self
.data
= None # force recompute
165 def __getitem__(self
, key
):
166 return self
.fields
[key
]
168 def __delitem__(self
, key
):
172 return self
.getData()
176 return len(self
.getData())
178 def pack(self
, format
, data
, field
= None):
180 print " pack( %s | %r | %s)" % (format
, data
, field
)
183 addressField
= self
.findAddressFieldFor(field
)
184 if (addressField
is not None) and (data
is None):
188 if format
[:1] == '_':
192 if format
[:1] == "'" or format
[:1] == '"':
196 two
= format
.split('=')
199 return self
.pack(two
[0], data
)
201 fields
= {'self':self
}
202 fields
.update(self
.fields
)
203 return self
.pack(two
[0], eval(two
[1], {}, fields
))
206 two
= format
.split('&')
209 return self
.pack(two
[0], data
)
211 if (self
.fields
.has_key(two
[1])) and (self
[two
[1]] is not None):
212 return self
.pack(two
[0], id(self
[two
[1]]) & ((1<<(calcsize(two
[0])*8))-1) )
214 return self
.pack(two
[0], 0)
217 two
= format
.split('-')
220 return self
.pack(two
[0],data
)
222 return self
.pack(two
[0], self
.calcPackFieldSize(two
[1]))
225 two
= format
.split('*')
229 answer
+= self
.pack(two
[1], each
)
232 if int(two
[0]) != len(data
):
233 raise Exception, "Array field has a constant size, and it doesn't match the actual value"
235 return self
.pack(two
[0], len(data
))+answer
238 # "printf" string specifier
239 if format
[:1] == '%':
240 # format string like specifier
244 if format
[:1] == 'z':
245 return str(data
)+'\0'
248 if format
[:1] == 'u':
249 return str(data
)+'\0\0' + (len(data
) & 1 and '\0' or '')
251 # DCE-RPC/NDR string specifier
252 if format
[:1] == 'w':
257 l
= pack('<L', len(data
)/2)
258 return '%s\0\0\0\0%s%s' % (l
,l
,data
)
261 raise Exception, "Trying to pack None"
264 if format
[:1] == ':':
267 # struct like specifier
268 return pack(format
, data
)
270 def unpack(self
, format
, data
, dataClassOrCode
= str, field
= None):
272 print " unpack( %s | %r )" % (format
, data
)
275 addressField
= self
.findAddressFieldFor(field
)
276 if addressField
is not None:
277 if not self
[addressField
]:
281 if format
[:1] == '_':
282 if dataClassOrCode
!= str:
283 fields
= {'self':self
, 'inputDataLeft':data
}
284 fields
.update(self
.fields
)
285 return eval(dataClassOrCode
, {}, fields
)
290 if format
[:1] == "'" or format
[:1] == '"':
293 raise Exception, "Unpacked data doesn't match constant value '%r' should be '%r'" % (data
, answer
)
297 two
= format
.split('&')
299 return self
.unpack(two
[0],data
)
302 two
= format
.split('=')
304 return self
.unpack(two
[0],data
)
307 two
= format
.split('-')
309 return self
.unpack(two
[0],data
)
312 two
= format
.split('*')
319 sofar
+= self
.calcUnpackSize(two
[0], data
)
320 number
= self
.unpack(two
[0], data
[:sofar
])
324 while number
and sofar
< len(data
):
325 nsofar
= sofar
+ self
.calcUnpackSize(two
[1],data
[sofar
:])
326 answer
.append(self
.unpack(two
[1], data
[sofar
:nsofar
], dataClassOrCode
))
331 # "printf" string specifier
332 if format
[:1] == '%':
333 # format string like specifier
338 if data
[-1] != '\x00':
339 raise Exception, ("%s 'z' field is not NUL terminated: %r" % (field
, data
))
340 return data
[:-1] # remove trailing NUL
344 if data
[-2:] != '\x00\x00':
345 raise Exception, ("%s 'u' field is not NUL-NUL terminated: %r" % (field
, data
))
346 return data
[:-2] # remove trailing NUL
348 # DCE-RPC/NDR string specifier
350 l
= unpack('<L', data
[:4])[0]
351 return data
[12:12+l
*2]
355 return dataClassOrCode(data
)
357 # struct like specifier
358 return unpack(format
, data
)[0]
360 def calcPackSize(self
, format
, data
, field
= None):
361 # # print " calcPackSize %s:%r" % (format, data)
363 addressField
= self
.findAddressFieldFor(field
)
364 if addressField
is not None:
365 if not self
[addressField
]:
369 if format
[:1] == '_':
373 if format
[:1] == "'" or format
[:1] == '"':
377 two
= format
.split('&')
379 return self
.calcPackSize(two
[0], data
)
382 two
= format
.split('=')
384 return self
.calcPackSize(two
[0], data
)
387 two
= format
.split('-')
389 return self
.calcPackSize(two
[0], data
)
392 two
= format
.split('*')
396 if int(two
[0]) != len(data
):
397 raise Exception, "Array field has a constant size, and it doesn't match the actual value"
399 answer
+= self
.calcPackSize(two
[0], len(data
))
402 answer
+= self
.calcPackSize(two
[1], each
)
405 # "printf" string specifier
406 if format
[:1] == '%':
407 # format string like specifier
408 return len(format
% data
)
411 if format
[:1] == 'z':
415 if format
[:1] == 'u':
417 return l
+ (l
& 1 and 3 or 2)
419 # DCE-RPC/NDR string specifier
420 if format
[:1] == 'w':
425 if format
[:1] == ':':
428 # struct like specifier
429 return calcsize(format
)
431 def calcUnpackSize(self
, format
, data
, field
= None):
433 print " calcUnpackSize( %s | %s | %r)" % (field
, format
, data
)
436 if format
[:1] == '_':
439 addressField
= self
.findAddressFieldFor(field
)
440 if addressField
is not None:
441 if not self
[addressField
]:
445 lengthField
= self
.findLengthFieldFor(field
)
446 return self
[lengthField
]
450 # XXX: Try to match to actual values, raise if no match
453 if format
[:1] == "'" or format
[:1] == '"':
457 two
= format
.split('&')
459 return self
.calcUnpackSize(two
[0], data
)
462 two
= format
.split('=')
464 return self
.calcUnpackSize(two
[0], data
)
467 two
= format
.split('-')
469 return self
.calcUnpackSize(two
[0], data
)
472 two
= format
.split('*')
479 answer
+= self
.calcUnpackSize(two
[0], data
)
480 number
= self
.unpack(two
[0], data
[:answer
])
484 answer
+= self
.calcUnpackSize(two
[1], data
[answer
:])
486 while answer
< len(data
):
487 answer
+= self
.calcUnpackSize(two
[1], data
[answer
:])
490 # "printf" string specifier
491 if format
[:1] == '%':
492 raise Exception, "Can't guess the size of a printf like specifier for unpacking"
495 if format
[:1] == 'z':
496 return data
.index('\x00')+1
499 if format
[:1] == 'u':
500 l
= data
.index('\x00\x00')
501 return l
+ (l
& 1 and 3 or 2)
503 # DCE-RPC/NDR string specifier
504 if format
[:1] == 'w':
505 l
= unpack('<L', data
[:4])[0]
509 if format
[:1] == ':':
512 # struct like specifier
513 return calcsize(format
)
515 def calcPackFieldSize(self
, fieldName
, format
= None):
517 format
= self
.formatForField(fieldName
)
519 return self
.calcPackSize(format
, self
[fieldName
])
521 def formatForField(self
, fieldName
):
522 for field
in self
.commonHdr
+self
.structure
:
523 if field
[0] == fieldName
:
525 raise Exception, ("Field %s not found" % fieldName
)
527 def findAddressFieldFor(self
, fieldName
):
528 descriptor
= '&%s' % fieldName
530 for field
in self
.commonHdr
+self
.structure
:
531 if field
[1][-l
:] == descriptor
:
535 def findLengthFieldFor(self
, fieldName
):
536 descriptor
= '-%s' % fieldName
538 for field
in self
.commonHdr
+self
.structure
:
539 if field
[1][-l
:] == descriptor
:
543 def zeroValue(self
, format
):
544 two
= format
.split('*')
547 return (self
.zeroValue(two
[1]),)*int(two
[0])
549 if not format
.find('*') == -1: return ()
550 if 's' in format
: return ''
551 if format
in ['z',':','u']: return ''
552 if format
== 'w': return '\x00\x00'
557 for field
in self
.commonHdr
+ self
.structure
:
558 self
[field
[0]] = self
.zeroValue(field
[1])
560 def dump(self
, msg
= None, indent
= 0):
562 if msg
is None: msg
= self
.__class
__.__name
__
566 for field
in self
.commonHdr
+self
.structure
:
569 fixedFields
.append(i
)
570 if isinstance(self
[i
], Structure
):
571 self
[i
].dump('%s%s:{' % (ind
,i
), indent
= indent
+ 4)
574 print "%s%s: {%r}" % (ind
,i
,self
[i
])
575 # Do we have remaining fields not defined in the structures? let's
577 remainingFields
= list(set(self
.fields
) - set(fixedFields
))
578 for i
in remainingFields
:
579 if isinstance(self
[i
], Structure
):
580 self
[i
].dump('%s%s:{' % (ind
,i
), indent
= indent
+ 4)
583 print "%s%s: {%r}" % (ind
,i
,self
[i
])
586 class _StructureTest
:
588 def create(self
,data
= None):
590 return self
.theClass(data
, alignment
= self
.alignment
)
592 return self
.theClass(alignment
= self
.alignment
)
597 testName
= self
.__class
__.__name
__
598 print "starting test: %s....." % testName
601 a
.dump("packing.....")
603 print "packed: %r" % a_str
604 print "unpacking....."
605 b
= self
.create(a_str
)
606 b
.dump("unpacked.....")
607 print "repacking....."
610 print "ERROR: original packed and repacked don't match"
611 print "packed: %r" % b_str
613 class _Test_simple(_StructureTest
):
614 class theClass(Structure
):
627 ('code1','>L=len(arr1)*2+0x1000'),
630 def populate(self
, a
):
631 a
['default'] = 'hola'
633 a
['int3'] = 0x45444342
635 a
['u1'] = 'hola'.encode('utf_16_le')
637 a
['arr1'] = (0x12341234,0x88990077,0x41414141)
638 # a['len1'] = 0x42424242
640 class _Test_fixedLength(_Test_simple
):
641 def populate(self
, a
):
642 _Test_simple
.populate(self
, a
)
643 a
['len1'] = 0x42424242
645 class _Test_simple_aligned4(_Test_simple
):
648 class _Test_nested(_StructureTest
):
649 class theClass(Structure
):
650 class _Inner(Structure
):
651 structure
= (('data', 'z'),)
654 ('nest1', ':', _Inner
),
655 ('nest2', ':', _Inner
),
659 def populate(self
, a
):
660 a
['nest1'] = _Test_nested
.theClass
._Inner
()
661 a
['nest2'] = _Test_nested
.theClass
._Inner
()
662 a
['nest1']['data'] = 'hola manola'
663 a
['nest2']['data'] = 'chau loco'
664 a
['int'] = 0x12345678
666 class _Test_Optional(_StructureTest
):
667 class theClass(Structure
):
675 def populate(self
, a
):
676 a
['Name'] = 'Optional test'
677 a
['List'] = (1,2,3,4)
679 class _Test_Optional_sparse(_Test_Optional
):
680 def populate(self
, a
):
681 _Test_Optional
.populate(self
, a
)
684 class _Test_AsciiZArray(_StructureTest
):
685 class theClass(Structure
):
692 def populate(self
, a
):
695 a
['array'] = ('hola','manola','te traje')
697 class _Test_UnpackCode(_StructureTest
):
698 class theClass(Structure
):
700 ('leni','<L=len(uno)*2'),
701 ('cuchi','_-uno','leni/2'),
706 def populate(self
, a
):
707 a
['uno'] = 'soy un loco!'
708 a
['dos'] = 'que haces fiera'
710 class _Test_AAA(_StructureTest
):
711 class theClass(Structure
):
714 ('iv', '!L=((init_vector & 0xFFFFFF) << 8) | ((pad & 0x3f) << 2) | (keyid & 3)'),
715 ('init_vector', '_','(iv >> 8)'),
716 ('pad', '_','((iv >>2) & 0x3F)'),
717 ('keyid', '_','( iv & 0x03 )'),
718 ('dataLen', '_-data', 'len(inputDataLeft)-4'),
723 def populate(self
, a
):
724 a
['init_vector']=0x01020304
725 #a['pad']=int('01010101',2)
726 a
['pad']=int('010101',2)
728 a
['data']="\xA0\xA1\xA2\xA3\xA4\xA5\xA6\xA7\xA8\xA9"
729 a
['icv'] = 0x05060708
730 #a['iv'] = 0x01020304
732 if __name__
== '__main__':
736 _Test_fixedLength().run()
738 print "cannot repack because length is bogus"
740 _Test_simple_aligned4().run()
742 _Test_Optional().run()
743 _Test_Optional_sparse().run()
744 _Test_AsciiZArray().run()
745 _Test_UnpackCode().run()