Ticket #171: parser.patch

archive/__init__.py

@@ -9 +9 @@
-from hachoir_parser.archive.rpm import RpmFile
-from hachoir_parser.archive.sevenzip import SevenZipParser
-from hachoir_parser.archive.mar import MarFile
-
+
+

archive/bzip2_parser.py

@@ -1 +1 @@
-"""
-BZIP2 archive file
-
-
+, Robert Xiao
-"""
-
-from hachoir_parser import Parser
-
-
-
+
+
+
+
+
+
-from hachoir_core.text_handler import textHandler, hexadecimal
+from hachoir_parser.archive.zlib import build_tree, HuffmanCode
-
-try:
-    from bz2 import BZ2Decompressor
@@ -27 +31 @@
-except ImportError:
-    has_deflate = False
-
+class ZeroTerminatedNumber(Field):
+    """Zero (bit) terminated number: e.g. 11110 is 4."""
+    def __init__(self, parent, name, description=None):
+        Field.__init__(self, parent, name, 0, description)
+
+        endian = self.parent.endian
+        stream = self.parent.stream
+        addr = self.absolute_address
+
+        value = 0
+        while True:
+            bit = stream.readBits(addr, 1, endian)
+            addr += 1
+            self._size += 1
+            if not bit:
+                break
+            value += 1
+        self.createValue = lambda: value
+
+def move_to_front(l, c):
+    l[:] = l[c:c+1] + l[0:c] + l[c+1:]
+
+class Bzip2Bitmap(FieldSet):
+    def __init__(self, parent, name, nb_items, start_index, *args, **kwargs):
+        FieldSet.__init__(self, parent, name, *args, **kwargs)
+        self.nb_items = nb_items
+        self.start_index = start_index
+
+    def createFields(self):
+        for i in xrange(self.start_index, self.start_index+self.nb_items):
+            yield Bit(self, "symbol_used[%i]"%i, "Is the symbol %i (%r) used?"%(i, chr(i)))
+
+class Bzip2Lengths(FieldSet):
+    def __init__(self, parent, name, symbols, *args, **kwargs):
+        FieldSet.__init__(self, parent, name, *args, **kwargs)
+        self.symbols = symbols
+
+    def createFields(self):
+        yield Bits(self, "start_length", 5)
+        length = self["start_length"].value
+        lengths = []
+        for i in xrange(self.symbols):
+            while True:
+                bit = Bit(self, "change_length[%i][]"%i, "Should the length be changed for symbol %i?"%i)
+                yield bit
+                if not bit.value:
+                    break
+                else:
+                    bit = Enum(Bit(self, "length_decrement[%i][]"%i, "Decrement the value?"), {True: "Decrement", False: "Increment"})
+                    yield bit
+                    if bit.value:
+                        length -= 1
+                    else:
+                        length += 1
+            lengths.append(length)
+        self.final_length = length
+        self.tree = build_tree(lengths)
+
+class Bzip2Selectors(FieldSet):
+    def __init__(self, parent, name, ngroups, *args, **kwargs):
+        FieldSet.__init__(self, parent, name, *args, **kwargs)
+        self.groups = range(ngroups)
+
+    def createFields(self):
+        for i in xrange(self["../selectors_used"].value):
+            field = ZeroTerminatedNumber(self, "selector_list[]")
+            move_to_front(self.groups, field.value)
+            field.realvalue = self.groups[0]
+            field._description = "MTF'ed selector index: raw value %i, real value %i"%(field.value, field.realvalue)
+            yield field
+
+class Bzip2Block(FieldSet):
+    def createFields(self):
+        yield textHandler(Bits(self, "blockheader", 48, "Block header"), hexadecimal)
+        if self["blockheader"].value != 0x314159265359: # pi
+            raise ParserError("Invalid block header!")
+        yield textHandler(UInt32(self, "crc32", "CRC32 for this block"), hexadecimal)
+        yield Bit(self, "randomized", "Is this block randomized?")
+        yield Bits(self, "orig_bwt_pointer", 24, "Starting pointer into BWT after untransform")
+        yield GenericVector(self, "huffman_used_map", 16, Bit, 'block_used', "Bitmap showing which blocks (representing 16 literals each) are in use")
+        symbols_used = []
+        for index, block_used in enumerate(self["huffman_used_map"].array('block_used')):
+            if block_used.value:
+                start_index = index*16
+                field = Bzip2Bitmap(self, "huffman_used_bitmap[%i]"%index, 16, start_index, "Bitmap for block %i (literals %i to %i) showing which symbols are in use"%(index, start_index, start_index + 15))
+                yield field
+                for i, used in enumerate(field):
+                    if used.value:
+                        symbols_used.append(start_index + i)
+        yield Bits(self, "huffman_groups", 3, "Number of different Huffman tables in use")
+        yield Bits(self, "selectors_used", 15, "Number of times the Huffman tables are switched")
+        yield Bzip2Selectors(self, "selectors_list", self["huffman_groups"].value)
+        trees = []
+        for group in xrange(self["huffman_groups"].value):
+            field = Bzip2Lengths(self, "huffman_lengths[]", len(symbols_used)+2)
+            yield field
+            trees.append(field.tree)
+        counter = 0
+        rle_run = 0
+        selector_tree = None
+        while True:
+            if counter%50 == 0:
+                select_id = self["selectors_list"].array("selector_list")[counter//50].realvalue
+                selector_tree = trees[select_id]
+            field = HuffmanCode(self, "huffman_code[]", selector_tree)
+            if field.realvalue in [0, 1]:
+                # RLE codes
+                if rle_run == 0:
+                    rle_power = 1
+                rle_run += (field.realvalue + 1) * rle_power
+                rle_power <<= 1
+                field._description = "RLE Run Code %i (for %r); Total accumulated run %i (Huffman Code %i)" % (field.realvalue, chr(symbols_used[0]), rle_run, field.value)
+            elif field.realvalue == len(symbols_used)+1:
+                field._description = "Block Terminator (%i) (Huffman Code %i)"%(field.realvalue, field.value)
+                yield field
+                break
+            else:
+                rle_run = 0
+                move_to_front(symbols_used, field.realvalue-1)
+                field._description = "Literal %r (value %i) (Huffman Code %i)"%(chr(symbols_used[0]), field.realvalue, field.value)
+            yield field
+            if field.realvalue == len(symbols_used)+1:
+                break
+            counter += 1
+
+class Bzip2Stream(FieldSet):
+    START_BLOCK = 0x314159265359 # pi
+    END_STREAM = 0x177245385090 # sqrt(pi)
+    def createFields(self):
+        end = False
+        while not end:
+            marker = self.stream.readBits(self.absolute_address + self.current_size, 48, self.endian)
+            if marker == self.START_BLOCK:
+                yield Bzip2Block(self, "block[]")
+            elif marker == self.END_STREAM:
+                yield textHandler(Bits(self, "stream_end", 48, "End-of-stream marker"), hexadecimal)
+                yield textHandler(UInt32(self, "crc32", "CRC32 for entire stream"), hexadecimal)
+                padding = paddingSize(self.current_size, 8)
+                if padding:
+                    yield PaddingBits(self, "padding[]", padding)
+                end = True
+            else:
+                raise ParserError("Invalid marker 0x%02X!"%marker)
+
-class Bzip2Parser(Parser):
-    PARSER_TAGS = {
-        "id": "bzip2",
@@ -37 +185 @@
-        "magic": (('BZh', 0),),
-        "description": "bzip2 archive"
-    }
-LITTLE
+BIG
-
-    def validate(self):
-        if self.stream.readBytes(0, 3) != 'BZh':
@@ -50 +198 @@
-        yield String(self, "id", 3, "Identifier (BZh)", charset="ASCII")
-        yield Character(self, "blocksize", "Block size (KB of memory needed to uncompress)")
-
-        yield UInt8(self, "blockheader", "Block header")
-        if self["blockheader"].value == 0x17:
-            yield String(self, "id2", 4, "Identifier2 (re8P)", charset="ASCII")
-            yield UInt8(self, "id3", "Identifier3 (0x90)")
-        elif self["blockheader"].value == 0x31:
-            yield String(self, "id2", 5, "Identifier 2 (AY&SY)", charset="ASCII")
-            if self["id2"].value != "AY&SY":
-                raise ParserError("Invalid identifier 2 (AY&SY)!")
-        else:
-            raise ParserError("Invalid block header!")
-        yield textHandler(UInt32(self, "crc32", "CRC32"), hexadecimal)
-
-        if self._size is None: # TODO: is it possible to handle piped input?
-            raise NotImplementedError
-
@@ -73 +209 @@
-                    break
-            else:
-                filename = None
-ytes(self, "file", size
+zip2Stream(self, "file", size=size*8
-            if has_deflate:
-                CompressedField(self, Bunzip2)
-                def createInputStream(**args):

archive/cab.py

@@ -1 +1 @@
-"""
-Microsoft Cabinet (CAB) archive.
-
-
+, Robert Xiao
-Creation date: 31 january 2007
+
+- Microsoft Cabinet SDK
+  http://msdn2.microsoft.com/en-us/library/ms974336.aspx
-"""
-
+
-from hachoir_parser import Parser
-from hachoir_core.field import (FieldSet, Enum,
-    CString, String,
-
+8, UInt
-    DateTimeMSDOS32, RawBytes)
-from hachoir_parser.common.msdos import MSDOSFileAttr16
-from hachoir_core.text_handler import textHandler, hexadecimal, filesizeHandler
-from hachoir_core.endian import LITTLE_ENDIAN
+from hachoir_core.tools import paddingSize
+from hachoir_core.stream import StringInputStream
+from hachoir_parser.archive.lzx import LZXStream, lzx_decompress
+from hachoir_parser.archive.zlib import DeflateBlock
-
-MAX_NB_FOLDER = 30
-
@@ -26 +32 @@
-
-class Folder(FieldSet):
-    def createFields(self):
-_data", "Offset of data
-cf_data
+set", "Offset to data (from file start)
+data_blocks", "Number of data blocks which are in this cabinet
-        yield Enum(Bits(self, "compr_method", 4, "Compression method"), COMPRESSION_NAME)
-
-
+
+
+
+
+
+
+
+
-
-    def createDescription(self):
-        text= "Folder: compression %s" % self["compr_method"].display
-!= COMPRESSION_NONE:
-)" % self["compr_level"].value
+in [2, 3]: # Quantum or LZX use compression level
+: window size %u)" % (self["compr_level"].value, 2**self["compr_level"].value)
-        return text
-
+class CabFileAttributes(FieldSet):
+    def createFields(self):
+        yield Bit(self, "readonly")
+        yield Bit(self, "hidden")
+        yield Bit(self, "system")
+        yield Bits(self, "reserved[]", 2)
+        yield Bit(self, "archive", "Has the file been modified since the last backup?")
+        yield Bit(self, "exec", "Run file after extraction?")
+        yield Bit(self, "name_is_utf", "Is the filename using UTF-8?")
+        yield Bits(self, "reserved[]", 8)
+
-class File(FieldSet):
-    def createFields(self):
-        yield filesizeHandler(UInt32(self, "filesize", "Uncompressed file size"))
-
-
+
+
+
+
+
-        yield DateTimeMSDOS32(self, "timestamp")
-
-
+
+
+
+
+
-
-    def createDescription(self):
-        return "File %s (%s)" % (
-            self["filename"].display, self["filesize"].display)
-
-class Reserved(FieldSet):
-    def createFields(self):
-        yield UInt32(self, "size")
-        size = self["size"].value
-        if size:
-            yield RawBytes(self, "data", size)
-
-class Flags(FieldSet):
-    static_size = 16
-    def createFields(self):
@@ -66 +88 @@
-        yield Bit(self, "has_reserved")
-        yield NullBits(self, "padding", 13)
-
+class FragmentGroup:
+    def __init__(self, parser):
+        self.items = []
+        self.parser = parser
+        self.args = {}
+
+    def add(self, item):
+        self.items.append(item)
+
+    def createInputStream(self):
+        # FIXME: Use lazy stream creation
+        data = []
+        for item in self.items:
+            data.append( item["rawdata"].value )
+        data = "".join(data)
+
+        # FIXME: Use smarter code to send arguments
+        self.args["compr_level"] = self.items[0].parent.parent.folder["compr_level"].value
+        tags = {"class": self.parser, "args": self.args}
+        tags = tags.iteritems()
+        return StringInputStream(data, "<fragment group>", tags=tags)
+
+class CustomFragment(FieldSet):
+    def __init__(self, parent, name, size, parser, description=None, group=None):
+        FieldSet.__init__(self, parent, name, description, size=size)
+        if not group:
+            group = FragmentGroup(parser)
+        self.field_size = size
+        self.group = group
+        self.group.add(self)
+
+    def createFields(self):
+        yield RawBytes(self, "rawdata", self.field_size//8)
+
+    def _createInputStream(self, **args):
+        return self.group.createInputStream()
+
+class DataBlock(FieldSet):
+    def __init__(self, *args, **kwargs):
+        FieldSet.__init__(self, *args, **kwargs)
+        size = (self["size"].value + 8) * 8 # +8 for header values
+        if self["/flags/has_reserved"].value:
+            size += self["/reserved_data_size"].value * 8
+        self._size = size
+
+    def createFields(self):
+        yield textHandler(UInt32(self, "crc32"), hexadecimal)
+        yield UInt16(self, "size")
+        yield UInt16(self, "uncompressed_size", "If this is 0, this block is continued in a subsequent cabinet")
+        if self["/flags/has_reserved"].value and self["/reserved_data_size"].value:
+            yield RawBytes(self, "reserved_data", self["/reserved_data_size"].value, "Per-datablock reserved area")
+        compr_method = self.parent.folder["compr_method"].value
+        if compr_method == 0: # Uncompressed
+            yield RawBytes(self, "data", self["size"].value, "Folder Data")
+            self.parent.uncompressed_data += self["data"].value
+        elif compr_method == 1: # MSZIP
+            yield String(self, "mszip_signature", 2, "MSZIP Signature (CK)")
+            yield DeflateBlock(self, "deflate_block", self.parent.uncompressed_data)
+            padding = paddingSize(self.current_size, 8)
+            if padding:
+                yield PaddingBits(self, "padding[]", padding)
+            self.parent.uncompressed_data = self["deflate_block"].uncomp_data
+        elif compr_method == 2: # Quantum
+            yield RawBytes(self, "compr_data", self["size"].value, "Compressed Folder Data")
+        elif compr_method == 3: # LZX
+            group = getattr(self.parent.folder, "lzx_group", None)
+            field = CustomFragment(self, "data", self["size"].value*8, LZXStream, "LZX data fragment", group)
+            self.parent.folder.lzx_group = field.group
+            yield field
+
+class FolderParser(Parser):
+    endian = LITTLE_ENDIAN
+    def createFields(self):
+        for file in sorted(self.files, key=lambda x:x["folder_offset"].value):
+            padding = self.seekByte(file["folder_offset"].value)
+            if padding:
+                yield padding
+            yield RawBytes(self, "file[]", file["filesize"].value, file.description)
+
+class FolderData(FieldSet):
+    def __init__(self, parent, name, folder, files, *args, **kwargs):
+        FieldSet.__init__(self, parent, name, *args, **kwargs)
+        def createInputStream(cis, source=None, **args):
+            stream = cis(source=source)
+            tags = args.setdefault("tags",[])
+            tags.extend(stream.tags)
+            tags.append(( "class", FolderParser ))
+            tags.append(( "args", {'files': files} ))
+            for unused in self:
+                pass
+            if folder["compr_method"].value == 3: # LZX
+                self.uncompressed_data = lzx_decompress(self["block[0]/data"].getSubIStream(), folder["compr_level"].value)
+            return StringInputStream(self.uncompressed_data, source=source, **args)
+        self.setSubIStream(createInputStream)
+        self.files = files
+        self.folder = folder # Folder fieldset
+
+    def createFields(self):
+        self.uncompressed_data = ""
+        for index in xrange(self.folder["data_blocks"].value):
+            block = DataBlock(self, "block[]")
+            for i in block:
+                pass
+            yield block
+
-class CabFile(Parser):
-    endian = LITTLE_ENDIAN
-    MAGIC = "MSCF"
@@ -82 +209 @@
-    def validate(self):
-        if self.stream.readBytes(0, 4) != self.MAGIC:
-            return "Invalid magic"
-cab_version"].value != 0x010
-s)" % self["cab_version"].display
+major_version"].value != 1 or self["minor_version"].value != 
+i.%i)" % (self["major_version"].value, self["minor_version"].value)
-        if not (1 <= self["nb_folder"].value <= MAX_NB_FOLDER):
-            return "Invalid number of folder (%s)" % self["nb_folder"].value
-        return True
@@ -95 +222 @@
-        yield textHandler(UInt32(self, "fld_checksum", "Folders checksum (0 if not used)"), hexadecimal)
-        yield UInt32(self, "off_file", "Offset of first file")
-        yield textHandler(UInt32(self, "files_checksum", "Files checksum (0 if not used)"), hexadecimal)
-
+
+
-        yield UInt16(self, "nb_folder", "Number of folders")
-        yield UInt16(self, "nb_files", "Number of files")
-        yield Flags(self, "flags")
-        yield UInt16(self, "setid")
-number
+cabinet_serial
-
-        # --- TODO: Support flags
-        if self["flags/has_reserved"].value:
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
-
-        for index in xrange(self["nb_folder"].value):
-            yield Folder(self, "folder[]")
-        for index in xrange(self["nb_files"].value):
-            yield File(self, "file[]")
-
+        folders = sorted(enumerate(self.array("folder")), key=lambda x:x[1]["offset"].value)
+
+        for i in xrange(len(folders)):
+            index, folder = folders[i]
+            padding = self.seekByte(folder["offset"].value)
+            if padding:
+                yield padding
+            files = []
+            for file in self.array("file"):
+                if file["folder_index"].value == index:
+                    files.append(file)
+            if i+1 == len(folders):
+                size = (self.size // 8) - folder["offset"].value
+            else:
+                size = (folders[i+1][1]["offset"].value) - folder["offset"].value
+            yield FolderData(self, "folder_data[%i]" % index, folder, files, size=size*8)
+
-        end = self.seekBit(self.size, "endraw")
-        if end:
-            yield end

archive/lzx.py

@@ +1 @@
+"""LZX data stream parser.
+
+Also includes a decompression function (slow!!) which can decompress
+LZX data stored in a Hachoir stream.
+
+Author: Robert Xiao
+Creation date: July 18, 2007
+"""
+from hachoir_parser import Parser
+from hachoir_core.field import (FieldSet,
+    UInt32, Bit, Bits, PaddingBits,
+    RawBytes)
+from hachoir_core.endian import BIG_ENDIAN, LITTLE_ENDIAN
+from hachoir_core.tools import paddingSize, alignValue
+from hachoir_parser.archive.zlib import build_tree, HuffmanCode, extend_data
+from hachoir_core.bits import str2long
+import new # for instancemethod
+
+LZX_ENDIAN = "BADC"
+
+def readLZXBits(self, address, nbits, endian):
+    def Flip16Bits(data):
+        """Flip adjacent bytes, so as to convert little-endian to big
+        and vice versa, over 16 bits"""
+        result = [] # faster to join than to += strings
+        assert len(data) % 2 == 0
+        while len(data) >= 2:
+            result.append(data[1::-1]) # [1::-1] is [1] + [0]
+            data = data[2:]
+        result.append(data)
+        return ''.join(result)
+
+    if endian in (BIG_ENDIAN, LITTLE_ENDIAN):
+        return self.oldReadBits(address, nbits, endian)
+
+    assert endian == LZX_ENDIAN
+    assert hasattr(self, "lzx_start")
+    # lzx_start is the # of bits from the start of the LZX block
+
+    address_from_start = address - self.lzx_start
+    words_from_start, remainder = divmod(address_from_start, 16)
+    complete_nbits = alignValue(remainder + nbits, 16)
+
+    unused, data, missing = self.read(words_from_start*16 + self.lzx_start, complete_nbits) # get a full multiple of 2 bytes
+    shift = remainder
+    if missing:
+        raise ReadStreamError(nbits, address)
+    data = Flip16Bits(data)
+    value = str2long(data, BIG_ENDIAN) # the flipping above gives BE data
+    value >>= len(data) * 8 - shift - nbits
+    return value & (1 << nbits) - 1
+
+class LZXPreTreeEncodedTree(FieldSet):
+    def __init__(self, parent, name, num_elements, *args, **kwargs):
+        FieldSet.__init__(self, parent, name, *args, **kwargs)
+        self.num_elements = num_elements
+        
+    def createFields(self):
+        for i in xrange(20):
+            yield Bits(self, "pretree_lengths[]", 4)
+        pre_tree = build_tree([x.value for x in self.array("pretree_lengths")])
+        if not hasattr(self.root, "lzx_tree_lengths_"+self.name):
+            self.lengths = [0] * self.num_elements
+            setattr(self.root, "lzx_tree_lengths_"+self.name, self.lengths)
+        else:
+            self.lengths = getattr(self.root, "lzx_tree_lengths_"+self.name)
+        i = 0
+        while i < self.num_elements:
+            field = HuffmanCode(self, "tree_code[]", pre_tree)
+            if field.realvalue <= 16:
+                self.lengths[i] = (self.lengths[i] - field.realvalue) % 17
+                field._description = "Literal tree delta length %i (new length value %i for element %i) (Huffman Code %i)" % (
+                        field.realvalue, self.lengths[i], i, field.value)
+                i += 1
+                yield field
+            elif field.realvalue == 17:
+                field._description = "Tree Code 17: Zeros for 4-19 elements (Huffman Code %i)" % field.value
+                yield field
+                extra = Bits(self, "extra[]", 4)
+                zeros = 4 + extra.value
+                extra._description = "Extra bits: zeros for %i elements (elements %i through %i)" % (zeros, i, i+zeros-1)
+                yield extra
+                self.lengths[i:i+zeros] = [0] * zeros
+                i += zeros
+            elif field.realvalue == 18:
+                field._description = "Tree Code 18: Zeros for 20-51 elements (Huffman Code %i)" % field.value
+                yield field
+                extra = Bits(self, "extra[]", 5)
+                zeros = 20 + extra.value
+                extra._description = "Extra bits: zeros for %i elements (elements %i through %i)" % (zeros, i, i+zeros-1)
+                yield extra
+                self.lengths[i:i+zeros] = [0] * zeros
+                i += zeros
+            elif field.realvalue == 19:
+                field._description = "Tree Code 19: Same code for 4-5 elements"
+                yield field
+                extra = Bits(self, "extra[]", 1)
+                run = 4 + extra.value
+                extra._description = "Extra bits: run for %i elements (elements %i through %i)" % (run, i, i+run-1)
+                yield extra
+                newfield = HuffmanCode(self, "extra_tree_code[]", pre_tree)
+                assert newfield.realvalue <= 16
+                self.lengths[i:i+run] = [(self.lengths[i] - newfield.realvalue) % 17] * run
+                i += run
+            
+class LZXBlock(FieldSet):
+    WINDOW_SIZE = {15:30,
+                   16:32,
+                   17:34,
+                   18:36,
+                   19:38,
+                   20:42,
+                   21:50}
+    POSITION_SLOTS = {0:(0,0,0),
+                      1:(1,1,0),
+                      2:(2,2,0),
+                      3:(3,3,0),
+                      4:(4,5,1),
+                      5:(6,7,1),
+                      6:(8,11,2),
+                      7:(12,15,2),
+                      8:(16,23,3),
+                      9:(24,31,3),
+                      10:(32,47,4),
+                      11:(48,63,4),
+                      12:(64,95,5),
+                      13:(96,127,5),
+                      14:(128,191,6),
+                      15:(192,255,6),
+                      16:(256,383,7),
+                      17:(384,511,7),
+                      18:(512,767,8),
+                      19:(768,1023,8),
+                      20:(1024,1535,9),
+                      21:(1536,2047,9),
+                      22:(2048,3071,10),
+                      23:(3072,4095,10),
+                      24:(4096,6143,11),
+                      25:(6144,8191,11),
+                      26:(8192,12287,12),
+                      27:(12288,16383,12),
+                      28:(16384,24575,13),
+                      29:(24576,32767,13),
+                      30:(32768,49151,14),
+                      31:(49152,65535,14),
+                      32:(65536,98303,15),
+                      33:(98304,131071,15),
+                      34:(131072,196607,16),
+                      35:(196608,262143,16),
+                      36:(262144,393215,17),
+                      37:(393216,524287,17),
+                      38:(524288,655359,17),
+                      39:(655360,786431,17),
+                      40:(786432,917503,17),
+                      41:(917504,1048575,17),
+                      42:(1048576,1179647,17),
+                      43:(1179648,1310719,17),
+                      44:(1310720,1441791,17),
+                      45:(1441792,1572863,17),
+                      46:(1572864,1703935,17),
+                      47:(1703936,1835007,17),
+                      48:(1835008,1966079,17),
+                      49:(1966080,2097151,17),
+                      }
+    def createFields(self):
+        yield Bits(self, "block_type", 3)
+        yield Bits(self, "block_size", 24)
+        self.uncompressed_size = self["block_size"].value
+        self.compression_level = self.root.compr_level
+        self.window_size = self.WINDOW_SIZE[self.compression_level]
+        if self["block_type"].value == 1: # Verbatim block
+            yield LZXPreTreeEncodedTree(self, "main_tree_start", 256)
+            yield LZXPreTreeEncodedTree(self, "main_tree_rest", self.window_size * 8)
+            main_tree = build_tree(self["main_tree_start"].lengths + self["main_tree_rest"].lengths)
+            yield LZXPreTreeEncodedTree(self, "length_tree", 249)
+            length_tree = build_tree(self["length_tree"].lengths)
+            current_decoded_size = 0
+            while current_decoded_size < self.uncompressed_size:
+                if current_decoded_size % 32768 == 0 and current_decoded_size != 0:
+                    padding = paddingSize(self.address + self.current_size, 16)
+                    if padding:
+                        yield PaddingBits(self, "padding[]", padding)
+                field = HuffmanCode(self, "main_code[]", main_tree)
+                if field.realvalue < 256:
+                    field._description = "Literal value %r (Huffman Code %i)" % (chr(field.realvalue), field.value)
+                    current_decoded_size += 1
+                    self.parent.uncompressed_data += chr(field.realvalue)
+                    yield field
+                    continue
+                position_header, length_header = divmod(field.realvalue - 256, 8)
+                info = self.POSITION_SLOTS[position_header]
+                if info[2] == 0:
+                    if info[0] == 0:
+                        position = self.parent.r0
+                        field._description = "Position Slot %i, Position [R0] (%i)" % (position_header, position)
+                    elif info[0] == 1:
+                        position = self.parent.r1
+                        self.parent.r1 = self.parent.r0
+                        self.parent.r0 = position
+                        field._description = "Position Slot %i, Position [R1] (%i)" % (position_header, position)
+                    elif info[0] == 2:
+                        position = self.parent.r2
+                        self.parent.r2 = self.parent.r0
+                        self.parent.r0 = position
+                        field._description = "Position Slot %i, Position [R2] (%i)" % (position_header, position)
+                    else:
+                        position = info[0] - 2
+                        self.parent.r2 = self.parent.r1
+                        self.parent.r1 = self.parent.r0
+                        self.parent.r0 = position
+                        field._description = "Position Slot %i, Position %i" % (position_header, position)
+                else:
+                    field._description = "Position Slot %i, Positions %i to %i" % (position_header, info[0] - 2, info[1] - 2, field.value)
+                if length_header == 7:
+                    field._description += ", Length Values 9 and up (Huffman Code %i)"%field.value
+                    yield field
+                    length_field = HuffmanCode(self, "length_code[]", length_tree)
+                    length = length_field.realvalue + 9
+                    length_field._description = "Length Code %i, total length %i (Huffman Code %i)" % (length_field.realvalue, length, length_field.value)
+                    yield length_field
+                else:
+                    field._description += ", Length Value %i (Huffman Code %i)"%(length_header + 2, field.value)
+                    yield field
+                    length = length_header + 2
+                if info[2]:
+                    extrafield = Bits(self, "position_extra[%s" % field.name.split('[')[1], info[2])
+                    position = extrafield.value + info[0] - 2
+                    self.parent.r2 = self.parent.r1
+                    self.parent.r1 = self.parent.r0
+                    self.parent.r0 = position
+                    extrafield._description = "Position Extra Bits (%i), total position %i"%(extrafield.value, position)
+                    yield extrafield
+                self.parent.uncompressed_data = extend_data(self.parent.uncompressed_data, length, position)
+                current_decoded_size += length
+        elif self["block_type"].value == 2: # Aligned offset block
+            pass
+        elif self["block_type"].value == 3: # Uncompressed block
+            padding = paddingSize(self.address + self.current_size, 16)
+            if padding:
+                yield PaddingBits(self, "padding[]", padding)
+            else:
+                yield PaddingBits(self, "padding[]", 16)
+            self.endian = LITTLE_ENDIAN
+            yield UInt32(self, "r[]", "New value of R0")
+            yield UInt32(self, "r[]", "New value of R1")
+            yield UInt32(self, "r[]", "New value of R2")
+            self.parent.r0 = self["r[0]"].value
+            self.parent.r1 = self["r[1]"].value
+            self.parent.r2 = self["r[2]"].value
+            yield RawBytes(self, "data", self.uncompressed_size)
+            self.parent.uncompressed_data+=self["data"].value
+            if self["block_size"].value % 2:
+                yield PaddingBits(self, "padding", 8)
+
+class LZXStream(Parser):
+    endian = LZX_ENDIAN
+    def createFields(self):
+        if not hasattr(self.stream, "oldReadBits"):
+            self.stream.oldReadBits = self.stream.readBits
+            self.stream.readBits = new.instancemethod(readLZXBits, self.stream, self.stream.__class__)
+        self.stream.lzx_start = 0
+        self.uncompressed_data = ""
+        self.r0 = 1
+        self.r1 = 1
+        self.r2 = 1
+        yield Bit(self, "filesize_indicator")
+        if self["filesize_indicator"].value:
+            yield UInt32(self, "filesize")
+        while self.current_size < self.size:
+            block = LZXBlock(self, "block[]")
+            yield block
+            if self.size - self.current_size < 16:
+                padding = paddingSize(self.address + self.current_size, 16)
+                if padding:
+                    yield PaddingBits(self, "padding[]", padding)
+                break
+
+def lzx_decompress(stream, window_bits):
+    data = LZXStream(stream)
+    data.compr_level = window_bits
+    for unused in data:
+        pass
+    return data.uncompressed_data

archive/mozilla_ar.py

@@ +1 @@
+"""MAR (Mozilla ARchive) parser
+
+Author: Robert Xiao
+Creation date: July 10, 2007
+
+"""
+
+from hachoir_core.endian import BIG_ENDIAN
+from hachoir_core.field import (RootSeekableFieldSet, FieldSet,
+    String, CString, UInt32, RawBytes)
+from hachoir_core.text_handler import displayHandler, filesizeHandler
+from hachoir_core.tools import humanUnixAttributes
+from hachoir_parser import HachoirParser
+
+class IndexEntry(FieldSet):
+    def createFields(self):
+        yield UInt32(self, "offset", "Offset in bytes relative to start of archive")
+        yield filesizeHandler(UInt32(self, "length", "Length in bytes"))
+        yield displayHandler(UInt32(self, "flags"), humanUnixAttributes)
+        yield CString(self, "name", "Filename (byte array)")
+    def createDescription(self):
+        return 'File %s, Size %s, Mode %s'%(
+            self["name"].display, self["length"].display, self["flags"].display)
+class MozillaArchive(HachoirParser, RootSeekableFieldSet):
+    MAGIC = "MAR1"
+    PARSER_TAGS = {
+        "id": "mozilla_ar",
+        "category": "archive",
+        "file_ext": ("mar",),
+        "min_size": (8+4+13)*8,  # Header, Index Header, 1 Index Entry
+        "magic": ((MAGIC, 0),),
+        "description": "Mozilla Archive",
+    }
+    endian = BIG_ENDIAN
+    
+    def __init__(self, stream, **args):
+        RootSeekableFieldSet.__init__(self, None, "root", stream, None, stream.askSize(self))
+        HachoirParser.__init__(self, stream, **args)
+        
+    def validate(self):
+        if self.stream.readBytes(0, 4) != self.MAGIC:
+            return "Invalid magic"
+        return True
+
+    def createFields(self):
+        yield String(self, "magic", 4, "File signature (MAR1)", charset="ASCII")
+        yield UInt32(self, "index_offset", "Offset to index relative to file start")
+        self.seekByte(self["index_offset"].value, False)
+        yield UInt32(self, "index_size", "size of index in bytes")
+        current_index_size = 0 # bytes
+        while current_index_size < self["index_size"].value:
+            # plus 4 compensates for index_size
+            self.seekByte(self["index_offset"].value + current_index_size + 4, False)
+            entry = IndexEntry(self, "index_entry[]")
+            yield entry
+            current_index_size += entry.size // 8
+            self.seekByte(entry["offset"].value, False)
+            yield RawBytes(self, "file[]", entry["length"].value)

archive/zlib.py

@@ +1 @@
+"""Detailed ZLIB parser
+
+Author: Robert Xiao
+Creation date: July 9 2007
+
+"""
+
+from hachoir_parser import Parser
+from hachoir_core.field import (Bit, Bits, Field, Int16, UInt32,
+    Enum, FieldSet, GenericFieldSet,
+    PaddingBits, ParserError, RawBytes)
+from hachoir_core.endian import LITTLE_ENDIAN
+from hachoir_core.text_handler import textHandler, hexadecimal
+from hachoir_core.tools import paddingSize, alignValue
+
+def extend_data(data, length, offset):
+    """Extend data using a length and an offset."""
+    if length >= offset:
+        new_data = data[-offset:] * (alignValue(length, offset) // offset)
+        return data + new_data[:length]
+    else:
+        return data + data[-offset:-offset+length]
+
+def build_tree(lengths):
+    max_length = max(lengths) + 1
+    bit_counts = [0]*max_length
+    next_code = [0]*max_length
+    tree = {}
+    for i in lengths:
+        if i:
+            bit_counts[i] += 1
+    code = 0
+    for i in xrange(1, len(bit_counts)):
+        next_code[i] = code = (code + bit_counts[i-1]) << 1
+    for i, ln in enumerate(lengths):
+        if ln:
+            tree[(ln, next_code[ln])] = i
+            next_code[ln] += 1
+    return tree</