]> jspc29.x-matter.uni-frankfurt.de Git - mvdsensorcontrol.git/commitdiff
mvd_unpacker: new Python module for new dataformat
authorPhilipp Klaus <klaus@physik.uni-frankfurt.de>
Wed, 22 Oct 2014 16:28:38 +0000 (18:28 +0200)
committerPhilipp Klaus <klaus@physik.uni-frankfurt.de>
Wed, 22 Oct 2014 16:28:38 +0000 (18:28 +0200)
tools/python_mvd_unpacker/.gitignore [new file with mode: 0644]
tools/python_mvd_unpacker/README.rst [new file with mode: 0644]
tools/python_mvd_unpacker/mvd_unpacker/__init__.py [new file with mode: 0644]
tools/python_mvd_unpacker/mvd_unpacker/unpacker.py [new file with mode: 0644]
tools/python_mvd_unpacker/mvd_unpacker/utils.py [new file with mode: 0644]
tools/python_mvd_unpacker/scripts/unpack_mvd [new file with mode: 0644]
tools/python_mvd_unpacker/setup.py [new file with mode: 0644]

diff --git a/tools/python_mvd_unpacker/.gitignore b/tools/python_mvd_unpacker/.gitignore
new file mode 100644 (file)
index 0000000..a00bee5
--- /dev/null
@@ -0,0 +1,59 @@
+*.txt
+
+### General Python stuff
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+bin/
+build/
+develop-eggs/
+dist/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# Rope
+.ropeproject
+
+# Django stuff:
+*.log
+*.pot
+
+# Sphinx documentation
+docs/_build/
+
+
diff --git a/tools/python_mvd_unpacker/README.rst b/tools/python_mvd_unpacker/README.rst
new file mode 100644 (file)
index 0000000..0db1ec3
--- /dev/null
@@ -0,0 +1,19 @@
+
+mvd_unpacker
+============
+
+mvd_unpacker is a Python module facilitating the interpretation
+and unpacking of data collected for the Micro-Vertex Detector
+MVD for the CBM experiment.
+
+Requirements
+------------
+
+This software requires hld_unpacker, a Python module to read HLD files.
+
+Resources
+---------
+
+* `Data Format Specification <http://jspc29.x-matter.uni-frankfurt.de/git/?p=mvd_docu.git;a=blob_plain;f=dataformat/main.pdf;hb=HEAD>`_
+* `Mimosa 26 User Manual <http://www.iphc.cnrs.fr/IMG/pdf/M26_UserManual_light.pdf>`_
+
diff --git a/tools/python_mvd_unpacker/mvd_unpacker/__init__.py b/tools/python_mvd_unpacker/mvd_unpacker/__init__.py
new file mode 100644 (file)
index 0000000..744ea20
--- /dev/null
@@ -0,0 +1,5 @@
+
+__all__ = ['read_ROC_Message']
+
+from .unpacker import read_ROC_Message
+
diff --git a/tools/python_mvd_unpacker/mvd_unpacker/unpacker.py b/tools/python_mvd_unpacker/mvd_unpacker/unpacker.py
new file mode 100644 (file)
index 0000000..1a94c66
--- /dev/null
@@ -0,0 +1,176 @@
+#!/usr/bin/env python
+
+"""
+Unpacker code for the CBM MVD
+
+Written for Python3+
+"""
+
+import struct
+import pdb
+import io
+
+from .utils import split_by_n, str_32bit_chunks
+
+#### ------ Definitions for M-26 data ------ ####
+
+WORDSIZE = 4 # four bytes per word
+
+#### ------ Class definitions representing frames ------ ####
+
+class Message(object):
+
+    HEADER_FMT = ""
+
+    def __init__(self, header_bytes, payload_bytes):
+        self.header_bytes = header_bytes
+        self.payload_bytes = payload_bytes
+        self.header_content = struct.unpack(self.HEADER_FMT, header_bytes)
+        self._set_properties(*self.header_content)
+        self._read_payload()
+        self._check()
+
+    def _read_payload(self):
+        pass
+
+    def _check(self):
+        pass
+
+class ROC_Message(Message):
+
+    HEADER_FMT = ">BBxB"
+
+    def __init__(self, header_bytes, payload_bytes):
+        self.header_bytes = header_bytes
+        self.payload_bytes = payload_bytes
+        self.header_content = struct.unpack(self.HEADER_FMT, header_bytes[:4])
+        self._set_properties(*self.header_content)
+        self._read_payload()
+        self._check()
+
+    def _set_properties(self, header_version, data_version, header_size):
+        self.header_version = header_version
+        self.data_version = data_version
+        self.header_size = header_size
+
+    def _read_payload(self):
+        # This message can analyze the payload of ROC Messages
+        # It splits the payload to frames
+
+        try:
+            data_class = DATA_VERSIONS[self.data_version]
+        except KeyError:
+            data_class = FrameMessage
+
+        self.frames = []
+        pos = 0
+        while True:
+            try:
+                size = struct.unpack('>H', self.payload_bytes[pos:pos+2])[0]
+            except struct.error:
+                break
+            frame_data = self.payload_bytes[pos:pos+(size+1)*4]
+            self.frames.append(data_class(self.data_version, frame_data))
+            pos += (size+1)*4
+            # check if we could read one more frame:
+            if len(self.payload_bytes[pos:pos+WORDSIZE]) == 0: break
+            if len(self.payload_bytes[pos:pos+WORDSIZE]) < WORDSIZE: pdb.set_trace()
+
+class FrameMessage(object):
+
+    def __init__(self, data_type, data_bytes):
+        self.data_type = data_type
+        self.data_bytes = data_bytes
+        self._interpret()
+
+    def _interpret(self):
+        pass
+
+class Frame_Message_01(FrameMessage):
+
+    def _interpret(self):
+        hc = struct.unpack(">HxBIIIIIHxx", self.data_bytes[:7*4])
+        self.header_content = hc
+        self.frame_length = hc[0]
+        self.sensor_id = hc[1]
+        self.status = hc[2]
+        self.debug = hc[3]
+        self.time = hc[4]
+        self.start = hc[5]
+        self.frame_num = hc[6]
+        self.frame_length_2 = hc[7]
+        # everything else is sensor_data
+        self.sensor_data = self.data_bytes[7*4:]
+
+class Frame_Message_02(FrameMessage):
+    pass
+
+class Frame_Message_C0(FrameMessage):
+
+    def _interpret(self):
+        hc = struct.unpack(">HxBI", self.data_bytes[:2*4])
+        self.header_content = hc
+        self.frame_length = hc[0]
+        self.sensor_id = hc[1]
+        self.threshold = hc[2] >> 24
+        self.bank = (hc[2] >> 20) & 0xF
+        self.row = (hc[2] >> 8) & 0xFFF
+        self.run = hc[2] & 0xFF
+        # everything else is testmode data
+        self.sensor_data = self.data_bytes[2*4:]
+
+    def __str__(self):
+        fmt = "static init: 0x{:08x}  format version: 0x{:04x}  testmode: 0x{:04x}\n"
+        out = fmt.format(self.init, self.format_version, self.testmode)
+        fmt = "id: 0x{:08x}  status:   0x{:08x}  h5:        0x{:08x}\n"
+        out += fmt.format(self.id, self.status, self.h5)
+        #fmt = "h6: 0x{:08x}  external: 0x{:08x}  frame_num: 0x{:08x}\n"
+        #out += fmt.format(self.h6, self.external, self.frame_num)
+        fmt = "h6: 0x{:08x}  threshold: 0x{:04x}  run: 0x{:04x}  row: 0x{:04x}\n"
+        out += fmt.format(self.h6, self.threshold, self.run, self.row)
+        out += "\n"
+        out += str_32bit_chunks(self.payload_bytes)
+        out += "\n"
+        return out
+
+class ROC_Header_v1(object):
+    def __init__(self, header_bytes):
+        self.header_bytes = header_bytes
+
+class RegularReadoutFrame(Message):
+
+    # the S-Curve Frame header consists of 8 words
+    HEADERSIZE = WORDSIZE * 8
+    HEADER_FMT = '>IHHIIIHHHH'
+
+    def _check(self):
+        assert self.header_content[0] == 0xFFFFFFFF
+        assert (self.header_content[1]>>4) == (self.header_content[2]>>4) # should be 0xAAA 0xCCC 0xEEE (prototype)
+
+    def _set_properties(self, init, id_0, id_1, status, debug_0, debug_1, timestamp_1, timestamp_2, timestamp_3, timestamp_4):
+        self.h_init = init
+        self.id_0 = id_0 & 0b1111
+        self.id_1 = id_1 & 0b1111
+        self.status = status
+        self.debug_0 = debug_0
+        self.debug_1 = debug_1
+        self.timestamp1 = timestamp1
+        self.timestamp2 = timestamp2
+        self.timestamp3 = timestamp3
+        self.timestamp4 = timestamp4
+
+DATA_VERSIONS = {
+  0x01: Frame_Message_01,
+  0x02: Frame_Message_02,
+  0xC2: Frame_Message_C0,
+  0x00: Frame_Message_C0,
+}
+
+#### ------ Functions to read data into the MVD / ROC Message classes ------ ####
+
+def read_ROC_Message(message_bytes):
+    if len(message_bytes) < 3: pdb.set_trace
+    header_size = int(message_bytes[3])
+    # split the message into header and payload
+    return ROC_Message(message_bytes[:(1+header_size)*4], message_bytes[(1+header_size)*4:])
+
diff --git a/tools/python_mvd_unpacker/mvd_unpacker/utils.py b/tools/python_mvd_unpacker/mvd_unpacker/utils.py
new file mode 100644 (file)
index 0000000..93057fb
--- /dev/null
@@ -0,0 +1,30 @@
+#!/usr/bin/python
+
+import codecs
+
+def split_by_n( seq, n ):
+    """A generator to divide a sequence into chunks of n units."""
+    #while seq:
+    #    yield seq[:n]
+    #    seq = seq[n:]
+    i = 0
+    while True:
+        part = seq[i:i+n]
+        if not len(part): break
+        yield part
+        i += n
+
+def str_32bit_chunks(data):
+    wordlength = 8 # in hex chars
+    words_per_line = 4
+    hexstr = codecs.encode(data, 'hex_codec').decode('ascii')
+    words = split_by_n(hexstr, wordlength)
+    words = ["0x" + word for word in words]
+    
+    i = 0
+    out = ""
+    for line_words in split_by_n(words, words_per_line):
+        out += "{:04x}:  {}\n".format(i, "  ".join(line_words))
+        i += int(wordlength/2) * words_per_line
+    return out
+
diff --git a/tools/python_mvd_unpacker/scripts/unpack_mvd b/tools/python_mvd_unpacker/scripts/unpack_mvd
new file mode 100644 (file)
index 0000000..a1fac8f
--- /dev/null
@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+
+'''
+CLI Unpacker for MVD data stored in HLD files
+'''
+
+from hld_unpacker import read
+from mvd_unpacker import read_ROC_Message
+from mvd_unpacker.utils import str_32bit_chunks
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser(description='Unpacker for HLD files containing M26 data.')
+    parser.add_argument('hldfile', metavar='HLD_FILE', type=argparse.FileType('rb'),
+          help='HLD file with M26 data to unpack')
+    args = parser.parse_args()
+
+    for event in read(args.hldfile):
+        for subevent in event.subevents:
+            for subsubevent in subevent.subsubevents:
+                if not len(subsubevent.payload_bytes): continue
+                if subsubevent.address == 0x5555: continue
+                roc_message = read_ROC_Message(subsubevent.payload_bytes)
+                print(roc_message.payload_bytes)
+                try:
+                    roc_message = read_ROC_Message(subsubevent.payload_bytes)
+                    print(roc_message.frames)
+                except:
+                    print("Not a ROC message (?):")
+                    print(str_32bit_chunks(subsubevent.payload_bytes))
+                    print("")
+
+if __name__ == "__main__":
+    main()
+
diff --git a/tools/python_mvd_unpacker/setup.py b/tools/python_mvd_unpacker/setup.py
new file mode 100644 (file)
index 0000000..0ea786f
--- /dev/null
@@ -0,0 +1,35 @@
+
+try:
+    from setuptools import setup
+except ImportError:
+    from distutils.core import setup
+
+setup(
+    name = 'mvd_unpacker',
+    version = '0.1.0',
+    description = '',
+    author = 'Philipp Klaus',
+    author_email = 'klaus@physik.uni-frankfurt.de',
+    packages = ['mvd_unpacker'],
+    scripts = [
+          'scripts/unpack_mvd',
+    ],
+    license = 'GPL',
+    long_description = "",
+    install_requires = [],
+    extras_require = {
+        'The command line tool unpack_mvd needs': ["hld_unpacker >= 0.1.0"],
+    },
+    keywords = 'MVD unpacker MIMOSA M26 CMOS CBM HEP',
+    classifiers = [
+        'Development Status :: 4 - Beta',
+        'Operating System :: OS Independent',
+        'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
+        'Programming Language :: Python',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.3',
+        'Topic :: System :: Hardware :: Hardware Drivers',
+        'Topic :: Scientific/Engineering',
+    ]
+)
+