From: Philipp Klaus Date: Wed, 22 Oct 2014 16:28:38 +0000 (+0200) Subject: mvd_unpacker: new Python module for new dataformat X-Git-Url: https://jspc29.x-matter.uni-frankfurt.de/git/?a=commitdiff_plain;h=956442360423690ba8772beabd3898f76f21a610;p=mvdsensorcontrol.git mvd_unpacker: new Python module for new dataformat --- diff --git a/tools/python_mvd_unpacker/.gitignore b/tools/python_mvd_unpacker/.gitignore new file mode 100644 index 0000000..a00bee5 --- /dev/null +++ b/tools/python_mvd_unpacker/.gitignore @@ -0,0 +1,59 @@ +*.txt + +### General Python stuff + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +bin/ +build/ +develop-eggs/ +dist/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.cache +nosetests.xml +coverage.xml + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Rope +.ropeproject + +# Django stuff: +*.log +*.pot + +# Sphinx documentation +docs/_build/ + + diff --git a/tools/python_mvd_unpacker/README.rst b/tools/python_mvd_unpacker/README.rst new file mode 100644 index 0000000..0db1ec3 --- /dev/null +++ b/tools/python_mvd_unpacker/README.rst @@ -0,0 +1,19 @@ + +mvd_unpacker +============ + +mvd_unpacker is a Python module facilitating the interpretation +and unpacking of data collected for the Micro-Vertex Detector +MVD for the CBM experiment. + +Requirements +------------ + +This software requires hld_unpacker, a Python module to read HLD files. + +Resources +--------- + +* `Data Format Specification `_ +* `Mimosa 26 User Manual `_ + diff --git a/tools/python_mvd_unpacker/mvd_unpacker/__init__.py b/tools/python_mvd_unpacker/mvd_unpacker/__init__.py new file mode 100644 index 0000000..744ea20 --- /dev/null +++ b/tools/python_mvd_unpacker/mvd_unpacker/__init__.py @@ -0,0 +1,5 @@ + +__all__ = ['read_ROC_Message'] + +from .unpacker import read_ROC_Message + diff --git a/tools/python_mvd_unpacker/mvd_unpacker/unpacker.py b/tools/python_mvd_unpacker/mvd_unpacker/unpacker.py new file mode 100644 index 0000000..1a94c66 --- /dev/null +++ b/tools/python_mvd_unpacker/mvd_unpacker/unpacker.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python + +""" +Unpacker code for the CBM MVD + +Written for Python3+ +""" + +import struct +import pdb +import io + +from .utils import split_by_n, str_32bit_chunks + +#### ------ Definitions for M-26 data ------ #### + +WORDSIZE = 4 # four bytes per word + +#### ------ Class definitions representing frames ------ #### + +class Message(object): + + HEADER_FMT = "" + + def __init__(self, header_bytes, payload_bytes): + self.header_bytes = header_bytes + self.payload_bytes = payload_bytes + self.header_content = struct.unpack(self.HEADER_FMT, header_bytes) + self._set_properties(*self.header_content) + self._read_payload() + self._check() + + def _read_payload(self): + pass + + def _check(self): + pass + +class ROC_Message(Message): + + HEADER_FMT = ">BBxB" + + def __init__(self, header_bytes, payload_bytes): + self.header_bytes = header_bytes + self.payload_bytes = payload_bytes + self.header_content = struct.unpack(self.HEADER_FMT, header_bytes[:4]) + self._set_properties(*self.header_content) + self._read_payload() + self._check() + + def _set_properties(self, header_version, data_version, header_size): + self.header_version = header_version + self.data_version = data_version + self.header_size = header_size + + def _read_payload(self): + # This message can analyze the payload of ROC Messages + # It splits the payload to frames + + try: + data_class = DATA_VERSIONS[self.data_version] + except KeyError: + data_class = FrameMessage + + self.frames = [] + pos = 0 + while True: + try: + size = struct.unpack('>H', self.payload_bytes[pos:pos+2])[0] + except struct.error: + break + frame_data = self.payload_bytes[pos:pos+(size+1)*4] + self.frames.append(data_class(self.data_version, frame_data)) + pos += (size+1)*4 + # check if we could read one more frame: + if len(self.payload_bytes[pos:pos+WORDSIZE]) == 0: break + if len(self.payload_bytes[pos:pos+WORDSIZE]) < WORDSIZE: pdb.set_trace() + +class FrameMessage(object): + + def __init__(self, data_type, data_bytes): + self.data_type = data_type + self.data_bytes = data_bytes + self._interpret() + + def _interpret(self): + pass + +class Frame_Message_01(FrameMessage): + + def _interpret(self): + hc = struct.unpack(">HxBIIIIIHxx", self.data_bytes[:7*4]) + self.header_content = hc + self.frame_length = hc[0] + self.sensor_id = hc[1] + self.status = hc[2] + self.debug = hc[3] + self.time = hc[4] + self.start = hc[5] + self.frame_num = hc[6] + self.frame_length_2 = hc[7] + # everything else is sensor_data + self.sensor_data = self.data_bytes[7*4:] + +class Frame_Message_02(FrameMessage): + pass + +class Frame_Message_C0(FrameMessage): + + def _interpret(self): + hc = struct.unpack(">HxBI", self.data_bytes[:2*4]) + self.header_content = hc + self.frame_length = hc[0] + self.sensor_id = hc[1] + self.threshold = hc[2] >> 24 + self.bank = (hc[2] >> 20) & 0xF + self.row = (hc[2] >> 8) & 0xFFF + self.run = hc[2] & 0xFF + # everything else is testmode data + self.sensor_data = self.data_bytes[2*4:] + + def __str__(self): + fmt = "static init: 0x{:08x} format version: 0x{:04x} testmode: 0x{:04x}\n" + out = fmt.format(self.init, self.format_version, self.testmode) + fmt = "id: 0x{:08x} status: 0x{:08x} h5: 0x{:08x}\n" + out += fmt.format(self.id, self.status, self.h5) + #fmt = "h6: 0x{:08x} external: 0x{:08x} frame_num: 0x{:08x}\n" + #out += fmt.format(self.h6, self.external, self.frame_num) + fmt = "h6: 0x{:08x} threshold: 0x{:04x} run: 0x{:04x} row: 0x{:04x}\n" + out += fmt.format(self.h6, self.threshold, self.run, self.row) + out += "\n" + out += str_32bit_chunks(self.payload_bytes) + out += "\n" + return out + +class ROC_Header_v1(object): + def __init__(self, header_bytes): + self.header_bytes = header_bytes + +class RegularReadoutFrame(Message): + + # the S-Curve Frame header consists of 8 words + HEADERSIZE = WORDSIZE * 8 + HEADER_FMT = '>IHHIIIHHHH' + + def _check(self): + assert self.header_content[0] == 0xFFFFFFFF + assert (self.header_content[1]>>4) == (self.header_content[2]>>4) # should be 0xAAA 0xCCC 0xEEE (prototype) + + def _set_properties(self, init, id_0, id_1, status, debug_0, debug_1, timestamp_1, timestamp_2, timestamp_3, timestamp_4): + self.h_init = init + self.id_0 = id_0 & 0b1111 + self.id_1 = id_1 & 0b1111 + self.status = status + self.debug_0 = debug_0 + self.debug_1 = debug_1 + self.timestamp1 = timestamp1 + self.timestamp2 = timestamp2 + self.timestamp3 = timestamp3 + self.timestamp4 = timestamp4 + +DATA_VERSIONS = { + 0x01: Frame_Message_01, + 0x02: Frame_Message_02, + 0xC2: Frame_Message_C0, + 0x00: Frame_Message_C0, +} + +#### ------ Functions to read data into the MVD / ROC Message classes ------ #### + +def read_ROC_Message(message_bytes): + if len(message_bytes) < 3: pdb.set_trace + header_size = int(message_bytes[3]) + # split the message into header and payload + return ROC_Message(message_bytes[:(1+header_size)*4], message_bytes[(1+header_size)*4:]) + diff --git a/tools/python_mvd_unpacker/mvd_unpacker/utils.py b/tools/python_mvd_unpacker/mvd_unpacker/utils.py new file mode 100644 index 0000000..93057fb --- /dev/null +++ b/tools/python_mvd_unpacker/mvd_unpacker/utils.py @@ -0,0 +1,30 @@ +#!/usr/bin/python + +import codecs + +def split_by_n( seq, n ): + """A generator to divide a sequence into chunks of n units.""" + #while seq: + # yield seq[:n] + # seq = seq[n:] + i = 0 + while True: + part = seq[i:i+n] + if not len(part): break + yield part + i += n + +def str_32bit_chunks(data): + wordlength = 8 # in hex chars + words_per_line = 4 + hexstr = codecs.encode(data, 'hex_codec').decode('ascii') + words = split_by_n(hexstr, wordlength) + words = ["0x" + word for word in words] + + i = 0 + out = "" + for line_words in split_by_n(words, words_per_line): + out += "{:04x}: {}\n".format(i, " ".join(line_words)) + i += int(wordlength/2) * words_per_line + return out + diff --git a/tools/python_mvd_unpacker/scripts/unpack_mvd b/tools/python_mvd_unpacker/scripts/unpack_mvd new file mode 100644 index 0000000..a1fac8f --- /dev/null +++ b/tools/python_mvd_unpacker/scripts/unpack_mvd @@ -0,0 +1,35 @@ +#!/usr/bin/env python + +''' +CLI Unpacker for MVD data stored in HLD files +''' + +from hld_unpacker import read +from mvd_unpacker import read_ROC_Message +from mvd_unpacker.utils import str_32bit_chunks + +def main(): + import argparse + parser = argparse.ArgumentParser(description='Unpacker for HLD files containing M26 data.') + parser.add_argument('hldfile', metavar='HLD_FILE', type=argparse.FileType('rb'), + help='HLD file with M26 data to unpack') + args = parser.parse_args() + + for event in read(args.hldfile): + for subevent in event.subevents: + for subsubevent in subevent.subsubevents: + if not len(subsubevent.payload_bytes): continue + if subsubevent.address == 0x5555: continue + roc_message = read_ROC_Message(subsubevent.payload_bytes) + print(roc_message.payload_bytes) + try: + roc_message = read_ROC_Message(subsubevent.payload_bytes) + print(roc_message.frames) + except: + print("Not a ROC message (?):") + print(str_32bit_chunks(subsubevent.payload_bytes)) + print("") + +if __name__ == "__main__": + main() + diff --git a/tools/python_mvd_unpacker/setup.py b/tools/python_mvd_unpacker/setup.py new file mode 100644 index 0000000..0ea786f --- /dev/null +++ b/tools/python_mvd_unpacker/setup.py @@ -0,0 +1,35 @@ + +try: + from setuptools import setup +except ImportError: + from distutils.core import setup + +setup( + name = 'mvd_unpacker', + version = '0.1.0', + description = '', + author = 'Philipp Klaus', + author_email = 'klaus@physik.uni-frankfurt.de', + packages = ['mvd_unpacker'], + scripts = [ + 'scripts/unpack_mvd', + ], + license = 'GPL', + long_description = "", + install_requires = [], + extras_require = { + 'The command line tool unpack_mvd needs': ["hld_unpacker >= 0.1.0"], + }, + keywords = 'MVD unpacker MIMOSA M26 CMOS CBM HEP', + classifiers = [ + 'Development Status :: 4 - Beta', + 'Operating System :: OS Independent', + 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Topic :: System :: Hardware :: Hardware Drivers', + 'Topic :: Scientific/Engineering', + ] +) +