From b238863bc8a523e96814f5cb1d97deff49c8db29 Mon Sep 17 00:00:00 2001 From: hadaq Date: Thu, 21 Feb 2019 11:34:14 +0100 Subject: [PATCH] update hmon doc files --- hmon/doc/daq-busy.htt | 12 +++++--- hmon/doc/daq-readout.htt | 0 hmon/doc/daq-spillcount.htt | 0 hmon/doc/ecal-count.htt | 0 hmon/doc/ecal-temp.htt | 0 hmon/doc/feeerr-goofy.htt | 0 hmon/doc/feeerr-trb.htt | 3 +- hmon/doc/hv-ecalhv.htt | 0 hmon/doc/hv-magnet.htt | 18 +++++++++++ hmon/doc/hv-mdchv.htt | 0 hmon/doc/hv-richhv.htt | 16 ++++++++++ hmon/doc/hv-sequencer.htt | 7 +++++ hmon/doc/main-rate.htt | 4 +-- hmon/doc/main-up.htt | 0 hmon/doc/mdc-linkqual.htt | 6 ++-- hmon/doc/misc-calib.htt | 18 +++++++++++ hmon/doc/misc-irq.htt | 0 hmon/doc/rate-pion1.htt | 0 hmon/doc/rate-pion2.htt | 0 hmon/doc/rate-pt1.htt | 0 hmon/doc/rich-bkpl.htt | 18 +++++++++++ hmon/doc/rich-curr.htt | 14 +++++++++ hmon/doc/rich-isob.htt | 7 +++++ hmon/doc/rich-temp.htt | 16 ++++++++++ hmon/doc/rich-volt.htt | 16 ++++++++++ hmon/doc/trg-pt1rate.htt | 9 ++++++ hmon/getebjson.pm | 61 +++++++++++++++++++++++++++++++++++++ hmon/hmon_calibration.pl | 42 +++++++++++++++++++++++++ utils/notifyall.sh | 44 +++++++++++++++++++++----- 29 files changed, 292 insertions(+), 19 deletions(-) create mode 100644 hmon/doc/daq-readout.htt create mode 100644 hmon/doc/daq-spillcount.htt create mode 100644 hmon/doc/ecal-count.htt create mode 100644 hmon/doc/ecal-temp.htt create mode 100644 hmon/doc/feeerr-goofy.htt create mode 100644 hmon/doc/hv-ecalhv.htt create mode 100644 hmon/doc/hv-magnet.htt create mode 100644 hmon/doc/hv-mdchv.htt create mode 100644 hmon/doc/hv-richhv.htt create mode 100644 hmon/doc/hv-sequencer.htt create mode 100644 hmon/doc/main-up.htt create mode 100644 hmon/doc/misc-calib.htt create mode 100644 hmon/doc/misc-irq.htt create mode 100644 hmon/doc/rate-pion1.htt create mode 100644 hmon/doc/rate-pion2.htt create mode 100644 hmon/doc/rate-pt1.htt create mode 100644 hmon/doc/rich-bkpl.htt create mode 100644 hmon/doc/rich-curr.htt create mode 100644 hmon/doc/rich-isob.htt create mode 100644 hmon/doc/rich-temp.htt create mode 100644 hmon/doc/rich-volt.htt create mode 100644 hmon/doc/trg-pt1rate.htt create mode 100644 hmon/getebjson.pm create mode 100644 hmon/hmon_calibration.pl diff --git a/hmon/doc/daq-busy.htt b/hmon/doc/daq-busy.htt index 9001235..4d24b07 100644 --- a/hmon/doc/daq-busy.htt +++ b/hmon/doc/daq-busy.htt @@ -6,12 +6,16 @@ During normal operation, the full message gives the busy times for all sub-syste

Error Handling if the button is not Green

-
Note that in some cases this is not the real source of the error - e.g. if the data transport to EB failed, the board with the smallest buffer and the most data gets busy first. +
Note that in some cases this is not the real source of the error - +e.g. if the data transport to EB failed, the board with the smallest +buffer and the most data gets busy first. Check also the next box "Read-out" for further information. diff --git a/hmon/doc/daq-readout.htt b/hmon/doc/daq-readout.htt new file mode 100644 index 0000000..e69de29 diff --git a/hmon/doc/daq-spillcount.htt b/hmon/doc/daq-spillcount.htt new file mode 100644 index 0000000..e69de29 diff --git a/hmon/doc/ecal-count.htt b/hmon/doc/ecal-count.htt new file mode 100644 index 0000000..e69de29 diff --git a/hmon/doc/ecal-temp.htt b/hmon/doc/ecal-temp.htt new file mode 100644 index 0000000..e69de29 diff --git a/hmon/doc/feeerr-goofy.htt b/hmon/doc/feeerr-goofy.htt new file mode 100644 index 0000000..e69de29 diff --git a/hmon/doc/feeerr-trb.htt b/hmon/doc/feeerr-trb.htt index 382f01a..6b4f2ed 100644 --- a/hmon/doc/feeerr-trb.htt +++ b/hmon/doc/feeerr-trb.htt @@ -4,8 +4,7 @@ This button shows if the TDC on the TRB are synchronized to the DAQ system.

Error Handling in case it is not Green

-If they are not synchronized there is the danger of mixing different events.
-You should restart DAQ several times.
+Some of the TRB2-based boards saw problems on the reference time input. There is the risk of event mixing. Restart DAQ. diff --git a/hmon/doc/hv-ecalhv.htt b/hmon/doc/hv-ecalhv.htt new file mode 100644 index 0000000..e69de29 diff --git a/hmon/doc/hv-magnet.htt b/hmon/doc/hv-magnet.htt new file mode 100644 index 0000000..e712229 --- /dev/null +++ b/hmon/doc/hv-magnet.htt @@ -0,0 +1,18 @@ +

Magnet

+

+This button shows the status of the Magnet systems. +

+

Error Handling in case it is not Green for > 2 min.

+ +

The script might show red precisely at midnight due to change of logfiles. Don't worry in this case. + +Call Torsten Heinz (mobile: 0175 388 4066 or home: 06162 982292 or work: 2781
+and Wolfgang Koenig (mobile: 0172 877 50 49 or home: 06071 35998 or work: 2720)! + + +

If no ssh connection to the magnet PC is possible (hadesp28), check if the machine is still running (upstairs, next to cryo) + + + + + diff --git a/hmon/doc/hv-mdchv.htt b/hmon/doc/hv-mdchv.htt new file mode 100644 index 0000000..e69de29 diff --git a/hmon/doc/hv-richhv.htt b/hmon/doc/hv-richhv.htt new file mode 100644 index 0000000..d8497d9 --- /dev/null +++ b/hmon/doc/hv-richhv.htt @@ -0,0 +1,16 @@ +

RICH High Voltage

+

+This button gives you the minimum and maximum Voltage of the HV of the RICH detector. +If this is off, the MAPMTs are not in measuring condition! +

Error Handling if the button is not Green

+ + + + + + + diff --git a/hmon/doc/hv-sequencer.htt b/hmon/doc/hv-sequencer.htt new file mode 100644 index 0000000..d76c428 --- /dev/null +++ b/hmon/doc/hv-sequencer.htt @@ -0,0 +1,7 @@ +

EPICS HV Sequencer

+ +

Some high voltage control is not working any more. Please inform the MDC operator before taking any action. +

Error Handling

+

Click the 'Restart Sequencer' button in the control window or run on lxhadeb06: +

ssh scs@lxhadeb06 'echo -en "\x18" | netcat -w 1 localhost 4813
+
Restarting might take a minute until the button is green again. diff --git a/hmon/doc/main-rate.htt b/hmon/doc/main-rate.htt index 89e0431..47df2f1 100644 --- a/hmon/doc/main-rate.htt +++ b/hmon/doc/main-rate.htt @@ -1,10 +1,10 @@

CTS Current Rate

-The current rate at which triggers are sent by the CTS. The rate should always be in the expected region!!! If it is not, first take a look at the CTS monitor for the TOF and RPC rates to find out where the problem is located. +The current rate at which triggers are sent by the CTS. The rate should always be in the expected region! If it is not, first take a look at the CTS monitor for the TOF and RPC rates to find out where the problem is located.

Error Handling

-
Shown trigger rate is low, but DAQ is running faster
The counter has a bug if the trigger rate is above 65 kHz (only reachable with pulser) due to a counter overflow. Ignore the warning message in this case. +
Shown trigger rate is low, but DAQ is running faster
The counter has a bug if the trigger rate is above 130kHz (only reachable with pulser) due to a counter overflow. Ignore the warning message in this case.
Rate is 2-5 Hz
Typically this happens when only calibration triggers are sent and no additional triggers are generated. Make sure this is desired and trigger settings are right.
The rate is 0
  • Check the "Busy Boards" monitor to see if a front-end has stopped working and is blocking the trigger. Follow the usual restart-DAQ guidelines to get the DAQ running again.
  • Check the CTS settings if the right trigger source is selected. diff --git a/hmon/doc/main-up.htt b/hmon/doc/main-up.htt new file mode 100644 index 0000000..e69de29 diff --git a/hmon/doc/mdc-linkqual.htt b/hmon/doc/mdc-linkqual.htt index 8f06cf6..559a59c 100644 --- a/hmon/doc/mdc-linkqual.htt +++ b/hmon/doc/mdc-linkqual.htt @@ -1,9 +1,7 @@

    MDC Link Errors

    -This button shows the number of link errors per second. If this number is too high, this could indicate that a front-end board -is damaged. No action on the DAQ is required. -

    -

    Problems can be related to unstable HV in some chambers. +This button shows the number of link errors per second. If this number is too high, this could indicate that a front-end board is damaged. Usually, this doesn't need any error handling. Problems can be related to unstable HV in some chambers. +

    If this number is huge and at the same time "trigger errors" also shows problems, there might be a problem with a front-end. Restart the DAQ - usually it will tell you that the compile time of one OEP is wrong, so you have to do a "Reload MDC". diff --git a/hmon/doc/misc-calib.htt b/hmon/doc/misc-calib.htt new file mode 100644 index 0000000..3f4c840 --- /dev/null +++ b/hmon/doc/misc-calib.htt @@ -0,0 +1,18 @@ +

    TDC Calibration status

    +

    +This button gives you the status of the last TDC calibration. + +

    Error Handling if the button is not Green

    + +Please run a new TDC calibration. Possible reasons are: +
      +
    • The temperatur has shifted to far from last calibration. +
    • There are front-ends in the system whose calibration file has not been updated +
    • The time since the last calibration is too long +
    • Eventbuilders return a bad quality of the last calibration run +
    + + + + + diff --git a/hmon/doc/misc-irq.htt b/hmon/doc/misc-irq.htt new file mode 100644 index 0000000..e69de29 diff --git a/hmon/doc/rate-pion1.htt b/hmon/doc/rate-pion1.htt new file mode 100644 index 0000000..e69de29 diff --git a/hmon/doc/rate-pion2.htt b/hmon/doc/rate-pion2.htt new file mode 100644 index 0000000..e69de29 diff --git a/hmon/doc/rate-pt1.htt b/hmon/doc/rate-pt1.htt new file mode 100644 index 0000000..e69de29 diff --git a/hmon/doc/rich-bkpl.htt b/hmon/doc/rich-bkpl.htt new file mode 100644 index 0000000..92a3cb4 --- /dev/null +++ b/hmon/doc/rich-bkpl.htt @@ -0,0 +1,18 @@ +

    DiRICH Temperature on Backplane

    +

    +This button gives you the minimum and maximum temperature between the Backplane and the MAPMTs (as well as at the inner spokes) in the RICH detector.
    +This temperatur is critical for the RICH! The backplane temperature is connected to a interlock that shuts down the RICH after a certain temperature is reached (See Tactical Overview). + +

    Error Handling if the button is not Green or Orange

    + +
      +
    • Temperature of DiRICH is too high. +
    • Check temperature of the Backplane and the Gas . Is update time up to date? +
    • If temperature is realy to high and it stays to high (ca. 2 min), call the operator. +
    • If Interlock has fired and RICH is down, call the operator. +
    + + + + + diff --git a/hmon/doc/rich-curr.htt b/hmon/doc/rich-curr.htt new file mode 100644 index 0000000..b46cd60 --- /dev/null +++ b/hmon/doc/rich-curr.htt @@ -0,0 +1,14 @@ +

    DiRICH Current

    +

    +This button gives you the minimum and maximum current of the FEE of the RICH detector. + + + + + + diff --git a/hmon/doc/rich-isob.htt b/hmon/doc/rich-isob.htt new file mode 100644 index 0000000..e7e5c64 --- /dev/null +++ b/hmon/doc/rich-isob.htt @@ -0,0 +1,7 @@ +

    Isobutan Gaspipe (RICH radiator)

    +

    +This value gives you the pressure of the inlet line of the Isobutan pipe near the Gassystems in front of the HADES Cave. + + + + diff --git a/hmon/doc/rich-temp.htt b/hmon/doc/rich-temp.htt new file mode 100644 index 0000000..2130bfd --- /dev/null +++ b/hmon/doc/rich-temp.htt @@ -0,0 +1,16 @@ +

    DiRICH Temperature

    +

    +This button gives you the minimum and maximum temperature of the DiRICH boards of the RICH detector. +This temperature is not directly criticly for the RICH operation, but it influences the inner temperature. +

    Error Handling if the button is not Green

    + +
      +
    • Temperature of DiRICH is too high. +
    • Check temperature of all DiRICHs. Is update time up to date. +
    • If temperature is realy to high and it stays to high (ca. 2 min), call the operator. +
    + + + + + diff --git a/hmon/doc/rich-volt.htt b/hmon/doc/rich-volt.htt new file mode 100644 index 0000000..008a780 --- /dev/null +++ b/hmon/doc/rich-volt.htt @@ -0,0 +1,16 @@ +

    DiRICH Voltage

    +

    +This button gives you the minimum and maximum voltage of the FEE of the RICH detector. +

    Error Handling if the button is not Green

    + +
      +
    • Check if a backplane has significant different voltage values in DiRICH 1.1V and DiRICH 2.5V. Higher values on 4x4 backplanes are +normal! +
    • If a single backplane is on low power and makes problems, the LV connector has to be pressed in. An expert is needed! +
    • If Backplanes are white, possibly a LV Powersupply failed. An expert is needed! +
    + + + + + diff --git a/hmon/doc/trg-pt1rate.htt b/hmon/doc/trg-pt1rate.htt new file mode 100644 index 0000000..d3e5978 --- /dev/null +++ b/hmon/doc/trg-pt1rate.htt @@ -0,0 +1,9 @@ +

    PT3 Rate

    +

    +This button shows the triggered high multiplicity events per second compared to the countrate in the Start-detector. +The second number should be around 0.7% during beam time since the interaction rate is around 1%. The first value is the rate +of PT3 triggers provided to the CTS. +
    +If something goes wrong, ask your shift leader. + + diff --git a/hmon/getebjson.pm b/hmon/getebjson.pm new file mode 100644 index 0000000..3e71cc1 --- /dev/null +++ b/hmon/getebjson.pm @@ -0,0 +1,61 @@ +#!/usr/bin/perl -w + +#Gets json files from EB, changes them to Hashes instead of arrays of names, and changes times to timestamps + +use strict; +use warnings; +use LWP::Simple qw(get); +use JSON qw( decode_json encode_json ); +use Data::Dumper; +use Try::Tiny; +use DateTime; +use DateTime::Format::ISO8601; + +sub hashify { + my $d = $_[0]; + if ($d->{'_childs'}) { + my $i = 0; + for my $c (0 .. scalar @{$d->{'_childs'}}) { + foreach my $k (keys %{$d->{'_childs'}[$i]}) { + $d->{$d->{'_childs'}[$i]{'_name'}}{$k} = $d->{'_childs'}[$i]{$k}; + if($k eq 'time') { + my $dt = DateTime::Format::ISO8601->parse_datetime($d->{$d->{'_childs'}[$i]{'_name'}}{$k}); + $d->{$d->{'_childs'}[$i]{'_name'}}{$k} = $dt->epoch(); + } + if($k eq '_childs') { + hashify($d->{$d->{'_childs'}[$i]{'_name'}}); + } + } + $i++; + } + delete ($d->{'_childs'}); + } + + } + +sub getjsonhash { + my ($src, $name) = @_; + my $baseurl; + if($src eq 'rawmon') {$baseurl = 'http://lxhadeb12:8090/';} + if($src eq 'ebmon') {$baseurl = 'http://lxhadeb12:8090/';} + if($src eq 'eb') {$baseurl = 'http://lxhadeb07:8099/';} + + my $rawdata = get ($baseurl.$name); + + my $data; + if (defined $rawdata) { + try{ + $data = decode_json($rawdata); + } + catch {sleep 5; return [];} + } + + + hashify($data); + return $data; + } + +# my $o = getjsonhash('eb', 'Master/BNET/get.json'); + + +1; diff --git a/hmon/hmon_calibration.pl b/hmon/hmon_calibration.pl new file mode 100644 index 0000000..f574eeb --- /dev/null +++ b/hmon/hmon_calibration.pl @@ -0,0 +1,42 @@ +#!/usr/bin/perl -w + +use warnings; +use strict; +use Time::HiRes qw( gettimeofday usleep time ); +use Data::Dumper; +use Hmon; +use QA; +use LWP::Simple; +use JSON qw( decode_json ); +use POSIX qw/floor ceil strftime/; +use lib '.'; +use getebjson; +use Storable qw(lock_store lock_retrieve); + + +my $fqa = QA::OpenQAFile(); + + + +while(1){ + my $data = getjsonhash('eb', 'Master/BNET/get.json'); + unless($data->{Inputs}{value}) {sleep 5; next;} + + + my $timesincecalib = time() - ($data->{LastCalibr}{'time'}); + my $txt = strftime("%d.%m. %H:%M",localtime($data->{LastCalibr}{'time'})); + my $longtext = "Last Calibration was done at ".$txt; + my $status = QA::GetQAState('below',$timesincecalib,@QA::TdcCalibrationInterval);; + + + QA::WriteQALog($fqa,"misc","calib",1200,$status,'Last TDC Calib',$txt,$longtext); + + + my $store; + + if($data->{LastCalibr}{'time'} != $store->{LastCalibr) { + + } + + sleep 60; + } diff --git a/utils/notifyall.sh b/utils/notifyall.sh index d5c066e..6355923 100755 --- a/utils/notifyall.sh +++ b/utils/notifyall.sh @@ -1,14 +1,44 @@ -#!/bin/bash +#!/usr/bin/perl + +use warnings; +use strict; + #ssh -X hadaq@hadesdaq01 " kdialog --title '$1' --passivepopup '$3: $2' 12 " 2>/dev/null 1>/dev/null& #ssh -X hadaq@hadesdaq01 " kdialog --title '$1' --passivepopup '$3: $2' 12 " & #ssh -X hadaq@hadesdaqp02 "DISPLAY=:0 kdialog --title '$1' --passivepopup '$3: $2' 12 " 2>/dev/null 1>/dev/null& -ssh -X hadaq@hadesdaqp03 "DISPLAY=:0 notify-send --expire-time=12000 '$1' '$3: $2'" 2>/dev/null 1>/dev/null& -ssh -X hadaq@hadesdaqp04 "DISPLAY=:0 notify-send --expire-time=12000 '$1' '$3: $2'" 2>/dev/null 1>/dev/null& +my $a = $ARGV[0]; +my $b = $ARGV[1]; +my $c = $ARGV[2]; + +$a=~s///g; +$b=~s/>//g; +$c=~s/>//g; + +my @list = qw(hadaq@hadesdaqp03:0 hadaq@hadesdaqp04:0 hadaq@hadesp30:1 hadaq@hadesp50:0); -ssh -X hadaq@hadesp30 "DISPLAY=:0 notify-send --expire-time=12000 '$1' '$3: $2'" 2>/dev/null 1>/dev/null& -ssh -X hadaq@hadesp30 "DISPLAY=:1 notify-send --expire-time=12000 '$1' '$3: $2'" 2>/dev/null 1>/dev/null& -ssh -X hadaq@hadesp50 "DISPLAY=:0 notify-send --expire-time=12000 '$1' '$3: $2'" 2>/dev/null 1>/dev/null& +foreach my $cur (@list) { + my ($server, $display) = split /:/, $cur; + my $command = qq|ssh -X $server "DISPLAY=:$display notify-send --expire-time=12000 '$a' '$c: $b'" 2>/dev/null 1>/dev/null&|; + #print $command . "\n"; + qx($command); +} + + +my $command = qq|logger -p local1.info -t "$ARGV[0]" "$ARGV[2]: $ARGV[1]"|; +qx($command); + +exit; + +#ssh -X hadaq@hadesdaqp03 "DISPLAY=:0 notify-send --expire-time=12000 '$1' '$3: $2'" 2>/dev/null 1>/dev/null& +#ssh -X hadaq@hadesdaqp04 "DISPLAY=:0 notify-send --expire-time=12000 '$1' '$3: $2'" 2>/dev/null 1>/dev/null& +#ssh -X hadaq@hadesp30 "DISPLAY=:0 notify-send --expire-time=12000 '$1' '$3: $2'" 2>/dev/null 1>/dev/null& +#ssh -X hadaq@hadesp30 "DISPLAY=:1 notify-send --expire-time=12000 '$1' '$3: $2'" 2>/dev/null 1>/dev/null& +#ssh -X hadaq@hadesp50 "DISPLAY=:0 notify-send --expire-time=12000 '$1' '$3: $2'" 2>/dev/null 1>/dev/null& #ssh -X hadaq@hadesp30 "DISPLAY=:0 kdialog --title '$1' --passivepopup '$3: $2' 12 " 2>/dev/null 1>/dev/null& @@ -16,5 +46,5 @@ ssh -X hadaq@hadesp50 "DISPLAY=:0 notify-send --expire-time=12000 '$1' '$3: #ssh -X hadaq@hadesp50 "DISPLAY=:0 kdialog --title '$1' --passivepopup '$3: $2' 12 " 2>/dev/null 1>/dev/null& #ssh -X hadaq@lxhadesdaqp " kdialog --title '$1' --passivepopup '$3: $2' 12 " 2>/dev/null 1>/dev/null & #ssh -X hadaq@lxhadesdaq " logger -p local1.info -t DAQ '$3 $2'" & -logger -p local1.info -t DAQ "$3 $2" + -- 2.43.0