New Python module: hld_unpacker

diff --git a/tools/python_hld_unpacker/.gitignore b/tools/python_hld_unpacker/.gitignore
new file mode 100644 (file)
index 0000000..a00bee5
--- /dev/null
+++ b/tools/python_hld_unpacker/.gitignore
@@ -0,0 +1,59 @@
+*.txt
+
+### General Python stuff
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+bin/
+build/
+develop-eggs/
+dist/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# Rope
+.ropeproject
+
+# Django stuff:
+*.log
+*.pot
+
+# Sphinx documentation
+docs/_build/
+
+

diff --git a/tools/python_hld_unpacker/README.rst b/tools/python_hld_unpacker/README.rst
new file mode 100644 (file)
index 0000000..0081e58
--- /dev/null
+++ b/tools/python_hld_unpacker/README.rst
@@ -0,0 +1,28 @@
+hld_unpacker
+============
+
+You can use this module as easily as
+
+::
+
+    #!/usr/bin/env python
+    
+    from hld_unpacker import read
+    
+    hldfile = open('./YOUR_FILE.hld', 'rb')
+    for event in read(hldfile):
+        print("{} {}".format(event.date, event.time))
+
+It also comes with a command line tool
+
+::
+
+    unpack_hld ./YOUR_FILE.hld
+
+References
+----------
+
+- `Jan Michel's PhD Thesis, section 5.1, pp. 63-65`__
+
+__ https://www-alt.gsi.de/documents/DOC-2012-Dec-83-1.pdf
+

diff --git a/tools/python_hld_unpacker/hld_unpacker/__init__.py b/tools/python_hld_unpacker/hld_unpacker/__init__.py
new file mode 100644 (file)
index 0000000..b52636b
--- /dev/null
+++ b/tools/python_hld_unpacker/hld_unpacker/__init__.py
@@ -0,0 +1,5 @@
+
+__all__ = ['read']
+
+from .unpacker import read
+

diff --git a/tools/python_hld_unpacker/hld_unpacker/unpacker.py b/tools/python_hld_unpacker/hld_unpacker/unpacker.py
new file mode 100644 (file)
index 0000000..751636a
--- /dev/null
+++ b/tools/python_hld_unpacker/hld_unpacker/unpacker.py
@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+
+"""
+Unpacker for HLD files
+
+Written for Python3+
+"""
+
+import struct
+import pdb
+import math
+import io
+
+from .utils import split_by_n, str_32bit_chunks
+
+#### ------ Size definitions for the HLD files ------ ####
+
+# we have 32bit words (i.e. 4 bytes per word)
+WORDSIZE = 4
+# The smallest unit to read is a block of 2 words (64bit)
+BLOCKSIZE =  WORDSIZE * 2
+
+#### ------ Class definitions representing (sub) events ------ ####
+
+class Event(object):
+
+    # the event header consists of 8 words
+    HEADERSIZE = WORDSIZE * 8
+
+    def __init__(self, header_bytes, payload_bytes):
+        self.header_bytes = header_bytes
+        self.payload_bytes = payload_bytes
+        struct_fmt = '<{:d}I'.format(Event.HEADERSIZE//WORDSIZE)
+        self.header_words = struct.unpack(struct_fmt, header_bytes)
+        self._set_properties(*self.header_words)
+        self.subevents = list(read_subevents(io.BytesIO(self.payload_bytes)))
+
+    def _set_properties(self, size, decoding, id, seqnr, date, time, runnr, expid):
+        self.size = size
+        self.decoding = decoding
+        self.id = id
+        self.seqnr = seqnr
+        self.date = date
+        self.time = time
+        self.runnr = runnr
+        self.expid = expid
+
+    def __str__(self):
+        fmt = "size:  0x{:08x}  decoding: 0x{:08x}  id:    0x{:08x}  seqnr: 0x{:08x}\n"
+        out  = fmt.format(self.size, self.decoding, self.id, self.seqnr)
+        fmt = "date:  0x{:08x}  time:     0x{:08x}  runnr: 0x{:08x}  expid: 0x{:08x}\n"
+        out += fmt.format(self.date, self.time, self.runnr, self.expid)
+        #out += "\n"
+        #out += str_32bit_chunks(self.payload_bytes)
+        #out += "\n"
+        for subevent in self.subevents:
+            out += str(subevent)
+        return out
+
+class SubEvent(object):
+
+    # the event header consists of 8 words
+    HEADERSIZE = WORDSIZE * 4
+
+    def __init__(self, header_bytes, payload_bytes):
+        self.header_bytes = header_bytes
+        self.payload_bytes = payload_bytes
+        struct_fmt = '>{:d}I'.format(SubEvent.HEADERSIZE//WORDSIZE)
+        self.header_words = struct.unpack(struct_fmt, header_bytes)
+        self._set_properties(*self.header_words)
+
+    def _set_properties(self, size, decoding, id, trignr):
+        self.size = size
+        self.decoding = decoding
+        self.id = id
+        self.trignr = trignr
+
+    def __str__(self):
+        fmt = "size:  0x{:08x}  decoding: 0x{:08x}  id:    0x{:08x}  trignr: 0x{:08x}\n"
+        out = fmt.format(self.size, self.decoding, self.id, self.trignr)
+        out += "\n"
+        out += str_32bit_chunks(self.payload_bytes)
+        return out
+
+#### ------ Generator Functions to read the HLD into the (sub)event classes ------ ####
+
+def read(hld_stream):
+    while True:
+        header_bytes = hld_stream.read(Event.HEADERSIZE)
+        if len(header_bytes) < Event.HEADERSIZE: break
+        # The size of the event in bytes is given in the first data word of the event.
+        # The size also includes the 4 bytes of its own data word:
+        size = struct.unpack('<I', header_bytes[:WORDSIZE])[0]
+        # We read already the full header, so we need to subtract 
+        actual_num_bytes = size - Event.HEADERSIZE
+        # The HLD file always writes blocks of length BLOCKSIZE and pads with 0x00 bytes at the end of blocks 
+        remaining_num_bytes = int(math.ceil(actual_num_bytes/BLOCKSIZE)*BLOCKSIZE)
+        data_bytes = hld_stream.read(remaining_num_bytes)
+        data_bytes = data_bytes[:actual_num_bytes]
+        #pdb.set_trace()
+        yield Event(header_bytes, data_bytes)
+
+def read_subevents(subevents_stream):
+    while True:
+        header_bytes = subevents_stream.read(SubEvent.HEADERSIZE)
+        if len(header_bytes) < SubEvent.HEADERSIZE: break
+        # The size of the event in bytes is given in the first data word of the event:
+        size = struct.unpack('>I', header_bytes[:WORDSIZE])[0]
+        # We read already all bytes belonging to the header, so we need to subtract them
+        actual_num_bytes = size - SubEvent.HEADERSIZE
+        remaining_num_bytes = int(math.ceil(actual_num_bytes/BLOCKSIZE)*BLOCKSIZE)
+        data_bytes = subevents_stream.read(remaining_num_bytes)
+        data_bytes = data_bytes[:actual_num_bytes]
+        #pdb.set_trace()
+        yield SubEvent(header_bytes, data_bytes)
+

diff --git a/tools/python_hld_unpacker/hld_unpacker/utils.py b/tools/python_hld_unpacker/hld_unpacker/utils.py
new file mode 100644 (file)
index 0000000..93057fb
--- /dev/null
+++ b/tools/python_hld_unpacker/hld_unpacker/utils.py
@@ -0,0 +1,30 @@
+#!/usr/bin/python
+
+import codecs
+
+def split_by_n( seq, n ):
+    """A generator to divide a sequence into chunks of n units."""
+    #while seq:
+    #    yield seq[:n]
+    #    seq = seq[n:]
+    i = 0
+    while True:
+        part = seq[i:i+n]
+        if not len(part): break
+        yield part
+        i += n
+
+def str_32bit_chunks(data):
+    wordlength = 8 # in hex chars
+    words_per_line = 4
+    hexstr = codecs.encode(data, 'hex_codec').decode('ascii')
+    words = split_by_n(hexstr, wordlength)
+    words = ["0x" + word for word in words]
+    
+    i = 0
+    out = ""
+    for line_words in split_by_n(words, words_per_line):
+        out += "{:04x}:  {}\n".format(i, "  ".join(line_words))
+        i += int(wordlength/2) * words_per_line
+    return out
+

diff --git a/tools/python_hld_unpacker/scripts/unpack_hld b/tools/python_hld_unpacker/scripts/unpack_hld
new file mode 100755 (executable)
index 0000000..bed8d67
--- /dev/null
+++ b/tools/python_hld_unpacker/scripts/unpack_hld
@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+
+'''
+CLI Unpacker for HLD files
+'''
+
+from hld_unpacker import read
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser(description='Unpacker for HLD files.')
+    parser.add_argument('hldfile', metavar='HLD_FILE', type=argparse.FileType('rb'),
+          help='HLD file to unpack')
+    args = parser.parse_args()
+
+    for event in read(args.hldfile):
+        print(event)
+
+if __name__ == "__main__":
+    main()
+

diff --git a/tools/python_hld_unpacker/setup.py b/tools/python_hld_unpacker/setup.py
new file mode 100644 (file)
index 0000000..436209b
--- /dev/null
+++ b/tools/python_hld_unpacker/setup.py
@@ -0,0 +1,30 @@
+
+try:
+    from setuptools import setup
+except ImportError:
+    from distutils.core import setup
+
+setup(
+    name = 'hld_unpacker',
+    version = '0.1.0',
+    description = 'An unpacker for HLD files.',
+    author = 'Philipp Klaus',
+    author_email = 'klaus@physik.uni-frankfurt.de',
+    packages = ['hld_unpacker'],
+    scripts = ['scripts/unpack_hld',],
+    license = 'GPL',
+    long_description = '',
+    install_requires = [],
+    keywords = 'HLD unpacker HADES CBM GSI HEP',
+    classifiers = [
+        'Development Status :: 4 - Beta',
+        'Operating System :: OS Independent',
+        'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
+        'Programming Language :: Python',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.3',
+        'Topic :: Scientific/Engineering',
+    ]
+)
+
+