From 15c189e8587a33774765a1725559b6864f031946 Mon Sep 17 00:00:00 2001 From: Philipp Klaus Date: Fri, 19 Sep 2014 15:47:52 +0200 Subject: [PATCH] New Python module: hld_unpacker --- tools/python_hld_unpacker/.gitignore | 59 +++++++++ tools/python_hld_unpacker/README.rst | 28 +++++ .../hld_unpacker/__init__.py | 5 + .../hld_unpacker/unpacker.py | 116 ++++++++++++++++++ .../python_hld_unpacker/hld_unpacker/utils.py | 30 +++++ tools/python_hld_unpacker/scripts/unpack_hld | 21 ++++ tools/python_hld_unpacker/setup.py | 30 +++++ 7 files changed, 289 insertions(+) create mode 100644 tools/python_hld_unpacker/.gitignore create mode 100644 tools/python_hld_unpacker/README.rst create mode 100644 tools/python_hld_unpacker/hld_unpacker/__init__.py create mode 100644 tools/python_hld_unpacker/hld_unpacker/unpacker.py create mode 100644 tools/python_hld_unpacker/hld_unpacker/utils.py create mode 100755 tools/python_hld_unpacker/scripts/unpack_hld create mode 100644 tools/python_hld_unpacker/setup.py diff --git a/tools/python_hld_unpacker/.gitignore b/tools/python_hld_unpacker/.gitignore new file mode 100644 index 0000000..a00bee5 --- /dev/null +++ b/tools/python_hld_unpacker/.gitignore @@ -0,0 +1,59 @@ +*.txt + +### General Python stuff + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +bin/ +build/ +develop-eggs/ +dist/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.cache +nosetests.xml +coverage.xml + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Rope +.ropeproject + +# Django stuff: +*.log +*.pot + +# Sphinx documentation +docs/_build/ + + diff --git a/tools/python_hld_unpacker/README.rst b/tools/python_hld_unpacker/README.rst new file mode 100644 index 0000000..0081e58 --- /dev/null +++ b/tools/python_hld_unpacker/README.rst @@ -0,0 +1,28 @@ +hld_unpacker +============ + +You can use this module as easily as + +:: + + #!/usr/bin/env python + + from hld_unpacker import read + + hldfile = open('./YOUR_FILE.hld', 'rb') + for event in read(hldfile): + print("{} {}".format(event.date, event.time)) + +It also comes with a command line tool + +:: + + unpack_hld ./YOUR_FILE.hld + +References +---------- + +- `Jan Michel's PhD Thesis, section 5.1, pp. 63-65`__ + +__ https://www-alt.gsi.de/documents/DOC-2012-Dec-83-1.pdf + diff --git a/tools/python_hld_unpacker/hld_unpacker/__init__.py b/tools/python_hld_unpacker/hld_unpacker/__init__.py new file mode 100644 index 0000000..b52636b --- /dev/null +++ b/tools/python_hld_unpacker/hld_unpacker/__init__.py @@ -0,0 +1,5 @@ + +__all__ = ['read'] + +from .unpacker import read + diff --git a/tools/python_hld_unpacker/hld_unpacker/unpacker.py b/tools/python_hld_unpacker/hld_unpacker/unpacker.py new file mode 100644 index 0000000..751636a --- /dev/null +++ b/tools/python_hld_unpacker/hld_unpacker/unpacker.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python + +""" +Unpacker for HLD files + +Written for Python3+ +""" + +import struct +import pdb +import math +import io + +from .utils import split_by_n, str_32bit_chunks + +#### ------ Size definitions for the HLD files ------ #### + +# we have 32bit words (i.e. 4 bytes per word) +WORDSIZE = 4 +# The smallest unit to read is a block of 2 words (64bit) +BLOCKSIZE = WORDSIZE * 2 + +#### ------ Class definitions representing (sub) events ------ #### + +class Event(object): + + # the event header consists of 8 words + HEADERSIZE = WORDSIZE * 8 + + def __init__(self, header_bytes, payload_bytes): + self.header_bytes = header_bytes + self.payload_bytes = payload_bytes + struct_fmt = '<{:d}I'.format(Event.HEADERSIZE//WORDSIZE) + self.header_words = struct.unpack(struct_fmt, header_bytes) + self._set_properties(*self.header_words) + self.subevents = list(read_subevents(io.BytesIO(self.payload_bytes))) + + def _set_properties(self, size, decoding, id, seqnr, date, time, runnr, expid): + self.size = size + self.decoding = decoding + self.id = id + self.seqnr = seqnr + self.date = date + self.time = time + self.runnr = runnr + self.expid = expid + + def __str__(self): + fmt = "size: 0x{:08x} decoding: 0x{:08x} id: 0x{:08x} seqnr: 0x{:08x}\n" + out = fmt.format(self.size, self.decoding, self.id, self.seqnr) + fmt = "date: 0x{:08x} time: 0x{:08x} runnr: 0x{:08x} expid: 0x{:08x}\n" + out += fmt.format(self.date, self.time, self.runnr, self.expid) + #out += "\n" + #out += str_32bit_chunks(self.payload_bytes) + #out += "\n" + for subevent in self.subevents: + out += str(subevent) + return out + +class SubEvent(object): + + # the event header consists of 8 words + HEADERSIZE = WORDSIZE * 4 + + def __init__(self, header_bytes, payload_bytes): + self.header_bytes = header_bytes + self.payload_bytes = payload_bytes + struct_fmt = '>{:d}I'.format(SubEvent.HEADERSIZE//WORDSIZE) + self.header_words = struct.unpack(struct_fmt, header_bytes) + self._set_properties(*self.header_words) + + def _set_properties(self, size, decoding, id, trignr): + self.size = size + self.decoding = decoding + self.id = id + self.trignr = trignr + + def __str__(self): + fmt = "size: 0x{:08x} decoding: 0x{:08x} id: 0x{:08x} trignr: 0x{:08x}\n" + out = fmt.format(self.size, self.decoding, self.id, self.trignr) + out += "\n" + out += str_32bit_chunks(self.payload_bytes) + return out + +#### ------ Generator Functions to read the HLD into the (sub)event classes ------ #### + +def read(hld_stream): + while True: + header_bytes = hld_stream.read(Event.HEADERSIZE) + if len(header_bytes) < Event.HEADERSIZE: break + # The size of the event in bytes is given in the first data word of the event. + # The size also includes the 4 bytes of its own data word: + size = struct.unpack('I', header_bytes[:WORDSIZE])[0] + # We read already all bytes belonging to the header, so we need to subtract them + actual_num_bytes = size - SubEvent.HEADERSIZE + remaining_num_bytes = int(math.ceil(actual_num_bytes/BLOCKSIZE)*BLOCKSIZE) + data_bytes = subevents_stream.read(remaining_num_bytes) + data_bytes = data_bytes[:actual_num_bytes] + #pdb.set_trace() + yield SubEvent(header_bytes, data_bytes) + diff --git a/tools/python_hld_unpacker/hld_unpacker/utils.py b/tools/python_hld_unpacker/hld_unpacker/utils.py new file mode 100644 index 0000000..93057fb --- /dev/null +++ b/tools/python_hld_unpacker/hld_unpacker/utils.py @@ -0,0 +1,30 @@ +#!/usr/bin/python + +import codecs + +def split_by_n( seq, n ): + """A generator to divide a sequence into chunks of n units.""" + #while seq: + # yield seq[:n] + # seq = seq[n:] + i = 0 + while True: + part = seq[i:i+n] + if not len(part): break + yield part + i += n + +def str_32bit_chunks(data): + wordlength = 8 # in hex chars + words_per_line = 4 + hexstr = codecs.encode(data, 'hex_codec').decode('ascii') + words = split_by_n(hexstr, wordlength) + words = ["0x" + word for word in words] + + i = 0 + out = "" + for line_words in split_by_n(words, words_per_line): + out += "{:04x}: {}\n".format(i, " ".join(line_words)) + i += int(wordlength/2) * words_per_line + return out + diff --git a/tools/python_hld_unpacker/scripts/unpack_hld b/tools/python_hld_unpacker/scripts/unpack_hld new file mode 100755 index 0000000..bed8d67 --- /dev/null +++ b/tools/python_hld_unpacker/scripts/unpack_hld @@ -0,0 +1,21 @@ +#!/usr/bin/env python + +''' +CLI Unpacker for HLD files +''' + +from hld_unpacker import read + +def main(): + import argparse + parser = argparse.ArgumentParser(description='Unpacker for HLD files.') + parser.add_argument('hldfile', metavar='HLD_FILE', type=argparse.FileType('rb'), + help='HLD file to unpack') + args = parser.parse_args() + + for event in read(args.hldfile): + print(event) + +if __name__ == "__main__": + main() + diff --git a/tools/python_hld_unpacker/setup.py b/tools/python_hld_unpacker/setup.py new file mode 100644 index 0000000..436209b --- /dev/null +++ b/tools/python_hld_unpacker/setup.py @@ -0,0 +1,30 @@ + +try: + from setuptools import setup +except ImportError: + from distutils.core import setup + +setup( + name = 'hld_unpacker', + version = '0.1.0', + description = 'An unpacker for HLD files.', + author = 'Philipp Klaus', + author_email = 'klaus@physik.uni-frankfurt.de', + packages = ['hld_unpacker'], + scripts = ['scripts/unpack_hld',], + license = 'GPL', + long_description = '', + install_requires = [], + keywords = 'HLD unpacker HADES CBM GSI HEP', + classifiers = [ + 'Development Status :: 4 - Beta', + 'Operating System :: OS Independent', + 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Topic :: Scientific/Engineering', + ] +) + + -- 2.43.0