From: Philipp Klaus <philipp.klaus@gmail.com>
Date: Tue, 14 Jul 2015 14:26:15 +0000 (+0200)
Subject: C-based unpacker, currently used for testmode QA
X-Git-Url: https://jspc29.x-matter.uni-frankfurt.de/git/?a=commitdiff_plain;h=5cc1a8d85941bf5d94350fca8e92b088b4bb0ff3;p=mvd_soft.git

C-based unpacker, currently used for testmode QA
---

diff --git a/s-curves/c_standalone/Makefile b/s-curves/c_standalone/Makefile
new file mode 100644
index 0000000..f13d05a
--- /dev/null
+++ b/s-curves/c_standalone/Makefile
@@ -0,0 +1,21 @@
+
+# http://mrbook.org/blog/tutorials/make/
+
+CC=gcc
+CFLAGS=-c -Wall -O3 -std=c11
+LDFLAGS=
+SOURCES=testmode_qa_unpacker.c hld.c math_helpers.c
+OBJECTS=$(SOURCES:.c=.o)
+EXECUTABLE=testmode_qa_unpacker
+
+all: $(SOURCES) $(EXECUTABLE)
+
+$(EXECUTABLE): $(OBJECTS)
+	$(CC) $(LDFLAGS) $(OBJECTS) -o $@
+
+.cpp.o:
+	$(CC) $(CFLAGS) $< -o $@
+
+clean:
+	-rm $(OBJECTS) $(EXECUTABLE)
+
diff --git a/s-curves/c_standalone/README.md b/s-curves/c_standalone/README.md
new file mode 100644
index 0000000..a3fc64c
--- /dev/null
+++ b/s-curves/c_standalone/README.md
@@ -0,0 +1,18 @@
+
+### Testmode Unpacker
+
+This is an unpacker for HLD files written in plain C.
+It is fast and versatile.
+
+A first command line tool to do QA on testmode files was
+created using this unpacker.
+
+#### Programming Conventions
+
+* Use tabs to indent (tab width : 2 spaces)
+
+#### Author
+
+* Philipp Klaus  
+  <klaus@physik.uni-frankfurt.de>
+
diff --git a/s-curves/c_standalone/hld.c b/s-curves/c_standalone/hld.c
new file mode 100644
index 0000000..f2893bf
--- /dev/null
+++ b/s-curves/c_standalone/hld.c
@@ -0,0 +1,88 @@
+#include <stdio.h>
+#include <stdint.h>
+#include <arpa/inet.h>
+
+#include "hld.h"
+
+int read_event(unsigned long *pos, FILE *ptr_file, int (*sensor_callback)(unsigned long *pos, FILE *ptr_file))
+{
+	unsigned long start_pos = *pos;
+	uint32_t header[EVENT_HEADERSIZE];
+	if (!fread(header, WORDSIZE, EVENT_HEADERSIZE, ptr_file)) return 1;
+	uint32_t size = header[0];
+	print2("EVENT Pos: 0x%lX Size: 0x%X\n", *pos, size);
+	*pos += EVENT_HEADERSIZE*WORDSIZE;
+	while (*pos - start_pos < size)
+		if (read_subevent(pos, ptr_file, sensor_callback)) break;
+	*pos = start_pos + size + size % BLOCKSIZE;
+	if (size % BLOCKSIZE != 0)
+		fseek(ptr_file, *pos, SEEK_SET);
+	return 0;
+}
+
+int read_subevent(unsigned long *pos, FILE *ptr_file, int (*sensor_callback)(unsigned long *pos, FILE *ptr_file))
+{
+	unsigned long start_pos = *pos;
+	uint32_t header[SUBEVENT_HEADERSIZE];
+	if (!fread(header, WORDSIZE, SUBEVENT_HEADERSIZE, ptr_file)) return 1;
+	uint32_t size = ntohl(header[0]);
+	print2("SUBEVENT Pos: 0x%lX Size: 0x%X\n", *pos, size);
+	*pos += SUBEVENT_HEADERSIZE*WORDSIZE;
+	while (*pos - start_pos < size)
+		if (read_subsubevent(pos, ptr_file, sensor_callback)) break;
+	*pos = start_pos + size + size % BLOCKSIZE;
+	if (size % BLOCKSIZE != 0)
+		fseek(ptr_file, *pos, SEEK_SET);
+	return 0;
+}
+
+int read_subsubevent(unsigned long *pos, FILE *ptr_file, int (*sensor_callback)(unsigned long *pos, FILE *ptr_file))
+{
+	unsigned long start_pos = *pos;
+	uint16_t header[SUBSUBEVENT_HEADERSIZE*2];
+	if (!fread(header, WORDSIZE/2, SUBSUBEVENT_HEADERSIZE*2, ptr_file)) return 1;
+	uint16_t size = ntohs(header[0]);
+	uint16_t address = ntohs(header[1]);
+	print2("SUBSUBEVENT Pos: 0x%lX Size: 0x%X Adress: 0x%X\n", *pos, size, address);
+	if (address >= 0xD000 && address < 0xE000)
+		read_roc(pos, ptr_file, sensor_callback);
+	*pos = start_pos + (size + SUBSUBEVENT_HEADERSIZE) * WORDSIZE;
+	fseek(ptr_file, *pos, SEEK_SET);
+	return 0;
+}
+
+int read_roc(unsigned long *pos, FILE *ptr_file, int (*sensor_callback)(unsigned long *pos, FILE *ptr_file))
+{
+	unsigned long start_pos = *pos;
+	uint8_t header[4];
+	uint8_t header_version, data_version, header_size;
+	if (!fread(header, 1, 4, ptr_file)) return 1;
+	header_version =	header[0];
+	data_version =		header[1];
+	header_size =			header[3];
+	BOOL frame_marked = FALSE;
+	uint32_t marker_timestamp, frame_timestamp = 0;
+	if (header_size >= 1)
+	{
+		if (!fread(&marker_timestamp, 4, 1, ptr_file)) return 1;
+		marker_timestamp = ntohl(marker_timestamp);
+		frame_marked = marker_timestamp & 0x80000000;
+		marker_timestamp &= 0x00FFFFFF;
+	}
+	if (header_size >= 2)
+	{
+		if (!fread(&frame_timestamp, 4, 1, ptr_file)) return 1;
+		frame_timestamp = ntohl(frame_timestamp);
+	}
+	print2("Marker: %d, Marker Timestamp: 0x%08X, Frame Timestamp: 0x%08X\n", frame_marked, marker_timestamp, frame_timestamp);
+	// finished reading the ROC header
+
+	// Starting to look at sensor header now
+	if (data_version == 0xC0)
+	{
+    sensor_callback(pos, ptr_file);
+	}
+	else return 1;
+	return 0;
+}
+
diff --git a/s-curves/c_standalone/hld.h b/s-curves/c_standalone/hld.h
new file mode 100644
index 0000000..89612c1
--- /dev/null
+++ b/s-curves/c_standalone/hld.h
@@ -0,0 +1,44 @@
+
+// HLD file format properties
+
+// size of a data word (32 bits) in bytes
+#define WORDSIZE 4
+// size of a data block (64 bits) in bytes
+#define BLOCKSIZE (WORDSIZE*2)
+// size of the headers in 32bit words
+#define EVENT_HEADERSIZE 8
+// size of the headers in 32bit words
+#define SUBEVENT_HEADERSIZE 4
+// size of the headers in 32bit words
+#define SUBSUBEVENT_HEADERSIZE 1
+
+// substituting the missing bool type
+#define FALSE   0
+#define TRUE    1
+#define BOOL uint8_t
+
+// debugging features
+#ifndef DEBUGLEVEL
+  #define DEBUGLEVEL 1
+
+  #if (DEBUGLEVEL >= 1)
+  #  define print1 printf
+  #else
+  #  define print1(...) (0)
+  #endif
+
+  #if (DEBUGLEVEL >= 2)
+  #  define print2 printf
+  #else
+  #  define print2(...) (0)
+  #endif
+#endif
+
+// for the FILE typedef
+#include <stdio.h>
+
+int read_event(unsigned long *pos, FILE *ptr_file, int (*sensor_callback)(unsigned long *pos, FILE *ptr_file));
+int read_subevent(unsigned long *pos, FILE *ptr_file, int (*sensor_callback)(unsigned long *pos, FILE *ptr_file));
+int read_subsubevent(unsigned long *pos, FILE *ptr_file, int (*sensor_callback)(unsigned long *pos, FILE *ptr_file));
+int read_roc(unsigned long *pos, FILE *ptr_file, int (*sensor_callback)(unsigned long *pos, FILE *ptr_file));
+
diff --git a/s-curves/c_standalone/math_helpers.c b/s-curves/c_standalone/math_helpers.c
new file mode 100644
index 0000000..6527515
--- /dev/null
+++ b/s-curves/c_standalone/math_helpers.c
@@ -0,0 +1,45 @@
+#include <string.h>
+#include "math_helpers.h"
+
+float mean(unsigned long n, unsigned long a[]) {
+	/*
+	Calculates the mean of the first
+	n values stored in the array a
+	*/
+	unsigned long sum = 0, i;
+	for(i=0; i<n; i++)
+		sum += a[i];
+	return((float) sum/n);
+}
+
+float median(unsigned long n, unsigned long x_orig[]) {
+	/*
+	Calculates the median of the first
+	n values stored in the array a
+	*/
+	float temp;
+	unsigned long i, j;
+	unsigned long x[n];
+	// create a copy of the array
+	memcpy(x, x_orig, n*sizeof(unsigned long));
+	// the following two loops sort the array x in ascending order
+	for(i=0; i<n-1; i++) {
+		for(j = i+1; j<n; j++) {
+			if(x[j] < x[i]) {
+				// swap elements
+				temp = x[i];
+				x[i] = x[j];
+				x[j] = temp;
+			}
+		}
+	}
+
+	if (n%2 == 0) {
+		// if there is an even number of elements, return mean of the two elements in the middle
+		return ((x[n/2] + x[n/2 - 1]) / 2.0);
+	} else {
+		// else return the element in the middle
+		return x[n/2];
+	}
+}
+
diff --git a/s-curves/c_standalone/math_helpers.h b/s-curves/c_standalone/math_helpers.h
new file mode 100644
index 0000000..d28d084
--- /dev/null
+++ b/s-curves/c_standalone/math_helpers.h
@@ -0,0 +1,4 @@
+
+float mean(unsigned long n, unsigned long a[]);
+float median(unsigned long n, unsigned long x[]);
+
diff --git a/s-curves/c_standalone/testmode_qa_unpacker.c b/s-curves/c_standalone/testmode_qa_unpacker.c
new file mode 100644
index 0000000..910bdc6
--- /dev/null
+++ b/s-curves/c_standalone/testmode_qa_unpacker.c
@@ -0,0 +1,248 @@
+#include <stdio.h>
+#include <stdint.h>
+#include <string.h>
+//#include <libgen.h>
+#include <arpa/inet.h>
+//#include <sys/types.h>
+//#include <sys/stat.h>
+//#include <unistd.h>
+
+#include "math_helpers.h"
+#include "testmode_qa_unpacker.h"
+#include "hld.h"
+
+// struct to hold the information from the testmode header
+typedef struct {
+	unsigned frame_length;
+	unsigned sensor_id;
+	unsigned threshold;
+	unsigned bank;
+	unsigned row;
+	unsigned run_number;
+} testmode_frame_header;
+
+// function to read the header of a testmode frame (data format version = 0xC0)
+testmode_frame_header read_C0_header(unsigned long *pos, FILE *ptr_file)
+{
+	uint32_t header[2];
+	fread(header, 4, 2, ptr_file);
+	header[0] = ntohl(header[0]);
+	header[1] = ntohl(header[1]);
+	testmode_frame_header h;
+	h.frame_length = header[0] >> 16;
+	h.sensor_id = header[0] & 0xFF;
+	h.threshold = header[1] >> 24;
+	h.bank = (header[1] & 0xF00000) >> 20;
+	h.row = (header[1] & 0xfff00 ) >> 8;
+	h.run_number = header[1] & 0xff;
+	return h;
+}
+
+// global variables to be filled by read_testmode_frame()
+testmode_frame_header h;
+uint32_t fired_pixels[36];
+
+// global variables to analyze the content in the main loop
+BOOL fresh_testmode_data = FALSE;
+testmode_frame_header min_values[BANKS];
+testmode_frame_header max_values[BANKS];
+uint8_t thresholds_seen[THRESHOLDS] = {0};
+unsigned fired_pixel_count[BANKS] = {0, 0, 0, 0};
+unsigned long total_fired_pixel_count[BANKS][BANKS] = {0}; // first index: bank under test
+unsigned count_entries_total_fired_pixel_count[BANKS] = {0};
+unsigned long row_occurence[BANKS][ROWS] = {0};
+
+// callback function to be passed to the HLD read_event() function
+int read_testmode_frame(unsigned long *pos, FILE *ptr_file)
+{
+	h = read_C0_header(pos, ptr_file);
+	if (!fread(fired_pixels, sizeof(uint32_t), 36, ptr_file)) return 1;
+	fresh_testmode_data = TRUE;
+	return 0;
+}
+
+int main(int argc, char **argv)
+{
+	unsigned long pos;
+	FILE *ptr_myfile;
+
+	if (argc < 2)
+	{
+		printf("Missing HLD file as argument\n");
+		return 1;
+	}
+	char *hld_filename = *++argv;
+	ptr_myfile = fopen(hld_filename, "rb");
+	if (!ptr_myfile)
+	{
+		printf("Unable to open file!\n");
+		return 1;
+	}
+
+	/*
+	// construct foldername for results
+	char results_foldername[1024] = "";
+	strcpy(results_foldername, hld_filename);
+	results_foldername[1023] = '\0'; // ensure it's a zero terminated string
+	char *dot = strrchr(results_foldername, '.');
+	if (dot && !strcmp(dot, ".hld"))
+		strcpy(dot, ".results");
+	else
+		strcat(results_foldername, ".results");
+	struct stat st = {0};
+	printf("Results foldername: %s\n", results_foldername);
+	if (stat(results_foldername, &st) == -1) {
+		mkdir(results_foldername, 0700);
+	}
+	*/
+
+	pos = 0;
+	memset(min_values, 0xff, BANKS*sizeof(testmode_frame_header));
+	memset(max_values, 0x00, BANKS*sizeof(testmode_frame_header));
+	unsigned long testmode_frame_no = 0;
+	unsigned long event_start_pos = pos;
+	BOOL may_cut = FALSE;
+	while (read_event(&pos, ptr_myfile, &read_testmode_frame) == 0)
+	{
+		if (!fresh_testmode_data)
+			continue;
+		fresh_testmode_data = FALSE;
+		print2("frame_length: 0x%02X, sensor_id: 0x%02X, threshold: 0x%02X, bank: %u, row: %u, run_number: %u\n",
+			h.frame_length, h.sensor_id, h.threshold, h.bank, h.row, h.run_number);
+		if (h.bank >= BANKS)
+		{
+			print1("This frame specifies the bank as %u which is too large. Maximum: %u", h.bank, BANKS-1);
+			return 1;
+		}
+		min_values[h.bank].frame_length = MIN(min_values[h.bank].frame_length, h.frame_length);
+		min_values[h.bank].sensor_id = MIN(min_values[h.bank].sensor_id, h.sensor_id);
+		min_values[h.bank].threshold = MIN(min_values[h.bank].threshold, h.threshold);
+		min_values[h.bank].row = MIN(min_values[h.bank].row, h.row);
+		min_values[h.bank].run_number = MIN(min_values[h.bank].run_number, h.run_number);
+		max_values[h.bank].frame_length = MAX(max_values[h.bank].frame_length, h.frame_length);
+		max_values[h.bank].sensor_id = MAX(max_values[h.bank].sensor_id, h.sensor_id);
+		max_values[h.bank].threshold = MAX(max_values[h.bank].threshold, h.threshold);
+		max_values[h.bank].row = MAX(max_values[h.bank].row, h.row);
+		max_values[h.bank].run_number = MAX(max_values[h.bank].run_number, h.run_number);
+		if (h.threshold >= THRESHOLDS)
+		{
+			print1("This frame specifies the threshold as %u which is too large in value. Maximum threshold: %u.\n", h.threshold, THRESHOLDS-1);
+			return 1;
+		}
+		if (!thresholds_seen[h.threshold])
+			print2("Threshold %u seen for the first time.\n", h.threshold);
+		thresholds_seen[h.threshold] = 1;
+		if (h.row >= ROWS)
+		{
+			print1("This frame specifies the row as %u which is too large in value. Maximum row #: %u.\n", h.row, ROWS-1);
+			print1("This happend in the event starting at byte pos 0x%lX\n", event_start_pos);
+			//h.row = h.row % ROWS;
+			may_cut = TRUE;
+			break;
+		}
+		for (int i = 0; i< 9*BANKS; i++)
+			fired_pixel_count[i/9] += __builtin_popcount(fired_pixels[i]);
+		for (int i = 0; i<BANKS; i++)
+			total_fired_pixel_count[h.bank][i] += fired_pixel_count[i];
+		count_entries_total_fired_pixel_count[h.bank] += 1;
+		print2("Fired pixel count by Matrix - A: %u B: %u C: %u D: %u\n",
+			fired_pixel_count[0], fired_pixel_count[1], fired_pixel_count[2], fired_pixel_count[3]);
+		row_occurence[h.bank][h.row] += 1;
+
+		// cleanup and increment for next loop
+		memset(fired_pixel_count, 0, sizeof(fired_pixel_count));
+		testmode_frame_no += 1;
+		event_start_pos = pos;
+
+	} // end while read_event()
+	if (may_cut)
+	{
+		char answer;
+		printf("\nWould you like to copy the .hld file up to this part to a new file? Enter Y or N: \n");
+		if (scanf(" %c", &answer) == 1 && (answer == 'Y' || answer == 'y') )
+		{
+			// construct foldername for results
+			char dest_hld_filename[1024] = "";
+			strcpy(dest_hld_filename, hld_filename);
+			dest_hld_filename[1023] = '\0'; // ensure it's a zero terminated string
+			char *dot = strrchr(dest_hld_filename, '.');
+			if (dot && !strcmp(dot, ".hld"))
+				strcpy(dot, ".cut.hld");
+			else
+				strcat(dest_hld_filename, ".cut.hld");
+			printf("Destination HLD filename: %s\n", dest_hld_filename);
+			fflush(stdout);
+			// -- copy parts of file
+			FILE *dest_fp;
+			char buffer[BLOCKSIZE];
+			if ((dest_fp = fopen(dest_hld_filename, "wb")) == NULL) {
+				fprintf(stderr, "Can't open %s\n", argv[2]);
+				fclose(dest_fp);
+				return 1;
+			}
+			int n = 1;
+			pos = 0;
+			fseek(ptr_myfile, pos, SEEK_SET);
+			while (n == 1 && pos < event_start_pos) {
+				n = fread(buffer, BLOCKSIZE, 1, ptr_myfile);
+				fwrite(buffer, BLOCKSIZE, 1, dest_fp);
+				pos += BLOCKSIZE;
+			}
+			fclose(dest_fp);
+			fclose(ptr_myfile);
+			printf("Done copying 0x%lX bytes to the new HLD file.\n", pos);
+			return 0;
+		}
+		else
+		{
+			printf("No. OK. I will finish now.\n");
+			return 1;
+		}
+	}
+	printf("\nTotal number of testmode frames: %lu\n\n", testmode_frame_no);
+	printf("frame_length MIN - A: %3u B: %3u C: %3u D: %3u\n",
+		min_values[0].frame_length, min_values[1].frame_length, min_values[2].frame_length, min_values[3].frame_length);
+	printf("             MAX - A: %3u B: %3u C: %3u D: %3u\n",
+		max_values[0].frame_length, max_values[1].frame_length, max_values[2].frame_length, max_values[3].frame_length);
+	printf("sensor_id    MIN - A: %3u B: %3u C: %3u D: %3u\n",
+		min_values[0].sensor_id, min_values[1].sensor_id, min_values[2].sensor_id, min_values[3].sensor_id);
+	printf("             MAX - A: %3u B: %3u C: %3u D: %3u\n",
+		max_values[0].sensor_id, max_values[1].sensor_id, max_values[2].sensor_id, max_values[3].sensor_id);
+	printf("threshold    MIN - A: %3u B: %3u C: %3u D: %3u\n",
+		min_values[0].threshold, min_values[1].threshold, min_values[2].threshold, min_values[3].threshold);
+	printf("             MAX - A: %3u B: %3u C: %3u D: %3u\n",
+		max_values[0].threshold, max_values[1].threshold, max_values[2].threshold, max_values[3].threshold);
+	printf("row          MIN - A: %3u B: %3u C: %3u D: %3u\n",
+		min_values[0].row, min_values[1].row, min_values[2].row, min_values[3].row);
+	printf("             MAX - A: %3u B: %3u C: %3u D: %3u\n",
+		max_values[0].row, max_values[1].row, max_values[2].row, max_values[3].row);
+	printf("run_number   MIN - A: %3u B: %3u C: %3u D: %3u\n",
+		min_values[0].run_number, min_values[1].run_number, min_values[2].run_number, min_values[3].run_number);
+	printf("             MAX - A: %3u B: %3u C: %3u D: %3u\n\n",
+		max_values[0].run_number, max_values[1].run_number, max_values[2].run_number, max_values[3].run_number);
+	// row_occurrence_median  |  it's fair enough to assume that the median is an integer here!
+	unsigned long rom = (unsigned long) median(BANKS*ROWS, row_occurence);
+	printf("Rows which don't occur with the median occurence of %lu:\n", rom);
+	for (unsigned row = 0; row < ROWS; row++)
+		if (row_occurence[0][row] != rom || row_occurence[1][row] != rom || row_occurence[2][row] != rom || row_occurence[3][row] != rom)
+			printf("Row %u A: %lu B: %lu C: %lu D: %lu\n", row, row_occurence[0][row], row_occurence[1][row], row_occurence[2][row], row_occurence[3][row]);
+	printf("\n");
+	printf("Thresholds seen:");
+	for (unsigned threshold = 0; threshold < THRESHOLDS; threshold++)
+		if (thresholds_seen[threshold]) printf(" %u", threshold);
+	printf("\n\n");
+
+	printf("AVG fired pixels of bank       A       B       C       D\n");
+	char first_bank = 0x41;
+	for (int i = 0; i<BANKS; i++)
+	{
+		printf("while measuring Bank %c: ", first_bank + i);
+		for (int j = 0; j<BANKS; j++)
+			printf(" %7.3f", ((double)total_fired_pixel_count[i][j]) / ((double) count_entries_total_fired_pixel_count[i]));
+		printf("\n");
+	}
+
+	fclose(ptr_myfile);
+
+	return 0;
+}
diff --git a/s-curves/c_standalone/testmode_qa_unpacker.h b/s-curves/c_standalone/testmode_qa_unpacker.h
new file mode 100644
index 0000000..55289cf
--- /dev/null
+++ b/s-curves/c_standalone/testmode_qa_unpacker.h
@@ -0,0 +1,39 @@
+
+// adding the missing bool type
+#define FALSE   0
+#define TRUE    1
+#define BOOL uint8_t
+
+// Add missing min() / max() functions:
+#define MIN(a,b) (((a)<(b))?(a):(b))
+#define MAX(a,b) (((a)>(b))?(a):(b))
+
+// debugging features
+#ifndef DEBUGLEVEL
+  #define DEBUGLEVEL 1
+
+  #if (DEBUGLEVEL >= 1)
+  #  define print1 printf
+  #else
+  #  define print1(...) (0)
+  #endif
+
+  #if (DEBUGLEVEL >= 2)
+  #  define print2 printf
+  #else
+  #  define print2(...) (0)
+  #endif
+#endif
+
+// sensor properties
+#define ROWS 576
+#define COLS 1152
+#define BANKS 4
+#define THRESHOLDS 256
+
+// for the FILE typedef
+#include <stdio.h>
+
+int read_testmode_frame(unsigned long *pos, FILE *ptr_file);
+int main(int argc, char **argv);
+