From: hadaq Date: Mon, 12 Aug 2024 13:53:00 +0000 (+0200) Subject: update hmon scripts X-Git-Url: https://jspc29.x-matter.uni-frankfurt.de/git/?a=commitdiff_plain;h=e72dc594b7cfa9b784aa9008270abf1fd3c01c6a;p=hadesdaq.git update hmon scripts --- diff --git a/hmon/QA.pm b/hmon/QA.pm index afc62c1..d12798b 100644 --- a/hmon/QA.pm +++ b/hmon/QA.pm @@ -48,8 +48,8 @@ our $entries->{'cats'} = ["main", $entries->{'main'} = ['time', 'rate','beamabort','up','spillcount']; $entries->{'daq'} = ['trbnet', 'timeouts', 'busy','readout','outofsync']; $entries->{'trg'} = ['spill', 'accepted', 'source','pt1rate', 'start']; -$entries->{'rate'} = ['pt1','pt2','pt3','','pt8']; -$entries->{'server'} = ['fill', 'cpu', 'ip', 'ebinputs', 'onlineqa']; #icinga, pwrsup +$entries->{'rate'} = ['pt1','pt2','pt3','pt8','veto']; +$entries->{'server'} = ['fill', 'cpu', '', 'ebinputs', 'onlineqa']; #icinga, pwrsup $entries->{'eb'} = ['run', 'rate','bytes', 'lostevt', 'errbits']; $entries->{'mdc'} = ['token', 'blocked', 'temp', 'linkqual', 'voltage']; $entries->{'endp'} = ['mdc','rich', 'tof', 'rpc', 'other']; @@ -96,10 +96,10 @@ our @RichInnerTempLimits = (37.5,38.5,39); our @TrgErrLimits = (100, 1000, 10000); our @CPULimits = (95, 100, 100); our @TimeoutLimits = (0, 0, 1); -our @TimeoutLimitsOnlyRICHMDC = (1, 5, 10); +our @TimeoutLimitsOnlyRICHMDC = (4, 6, 10); our @LinkErrLimits = (50, 500, 1000); our @MdcEndpMissingLimits = (4, 5, 6); #(0, 4, 5)!!! Oh dear. -our @RichEndpMissingLimits = (0, 3, 4); +our @RichEndpMissingLimits = (1, 3, 4); our @TofEndpMissingLimits = (0, 0, 0); our @RpcEndpMissingLimits = (0, 0, 0); our @OtherEndpMissingLimits = (0, 0, 0); @@ -140,7 +140,7 @@ our $PadiwaHodoNumber = 2; our @PadiwaHodoLimits = (0, 0, 0); our @LimitTriggerPerSpill = (1000, 0, 0); -our $AcceleratorCycle = 9; +our $AcceleratorCycle = 20; use constant {CTSAddress => 0x0003}; ############################################################################### @@ -165,7 +165,7 @@ our $QAServerOff = 0; # Missing Boards ############################################################################### -our @mdc_boards_removed =(0x2203);#(0x2203, 0x2027, 0x2057); #(0x2257, 0x2029); # 2233 added 2014-08-28 , 2203 at 2014/09/03, 2029 2014/09/23 +our @mdc_boards_removed =(); # 2333 added 2024-03-01 #(0x2203, 0x2027, 0x2057); #(0x2257, 0x2029); # 2233 added 2014-08-28 , 2203 at 2014/09/03, 2029 2014/09/23 our @mdc_chambers_removed =(); our @rich_boards_removed =();#(0x71a4,0x71a5,0x71b4,0x71b5,0x72a4,0x72a5,0x72b4,0x72b5,0x73a4,0x73a5,0x73b4,0x73b5,0x826a); our @tof_boards_removed =(); @@ -341,6 +341,30 @@ sub SciNotation { } } +# Tries to nicely format an integer +sub SciNotation2 { + my $v = shift; + return "undef" if (!defined $v); + return "0" if $v == 0; +# print $v."\n"; + if(abs($v) >= 1) { + return sprintf("%.1f", $v) if (abs($v) < 10) ; + return sprintf("%i", $v) if (abs($v) < 1000) ; + return sprintf("%.1f×103", $v / 1000.) if (abs($v) < 20000) ; + return sprintf("%i×103", $v / 1000.) if (abs($v) < 1E6) ; + return sprintf("%.1f×106", $v / 1000000.) if (abs($v) < 20E6) ; + return sprintf("%i×106", $v / 1000000.) if (abs($v) < 1E9) ; + return sprintf("%.2f×109", $v / 1E9) if (abs($v) < 20E9) ; + return sprintf("%.1f×109", $v / 1E9) if (abs($v) < 100E9) ; + return sprintf("%i",$v); + } + else { + return sprintf("%i×10-9", $v*1E9) if (abs($v) < 1E-6) ; + return sprintf("%i×10-6", $v*1E6) if (abs($v) < 1E-3) ; + return sprintf("%.1f×10-3", $v*1E3); + } +} + ############################################ # A simple date string sub getTimeString { diff --git a/hmon/doc/misc-magnet.htt b/hmon/doc/misc-magnet.htt index 1f4785c..20ef0b3 100644 --- a/hmon/doc/misc-magnet.htt +++ b/hmon/doc/misc-magnet.htt @@ -6,7 +6,7 @@ This button shows the status of the Magnet system.

In case of an error wait for the next update after ~ 30 seconds. Especially if a value reads back with a value of 0 it's likely not an error on the magnet but with getting the information from EPICS.

Error Handling in case it is not Green for > 2 min.

-

Call Torsten Heinz (mobile: 0175 388 4066 or home: 06162 982292 or work: 1818
+

Call Torsten Heinz (mobile: 0176 579 39435 or home: 06162 982292 or work: 1818
diff --git a/hmon/doc/misc-seu.htt b/hmon/doc/misc-seu.htt index 71d4b69..bcf8de7 100644 --- a/hmon/doc/misc-seu.htt +++ b/hmon/doc/misc-seu.htt @@ -6,9 +6,11 @@ front-ends (the most exposed systems!) can be included here.

Error Handling

+There's no need to react immediately, but some steps should be taken whenever the next +DAQ restart is needed. +Doing so before an actual failure of a board is always simpler and quicker. -The full message contains the addresses of affected boards - consider rebooting the -coresponding subsystem at your convenience. Doing so before an actual failure of -the board is always simpler and quicker. +

Suggestion: Use reboot "Quite-a-lot" first, and then see if any reported errors remain. +

The full message contains the addresses of affected boards - consider rebooting the +coresponding subsystem at your convenience. -
If many RICH Combiner boards show up, do a reboot RICH and reboot RICH combiners before the next DAQ restart. diff --git a/hmon/doc/misc2-tdctot.htt b/hmon/doc/misc2-tdctot.htt index e69de29..2a28d29 100644 --- a/hmon/doc/misc2-tdctot.htt +++ b/hmon/doc/misc2-tdctot.htt @@ -0,0 +1,8 @@ +

TDC Monitoring

+Checks the correct order of data words in TDCs. This sometimes can get corrupted with the "wrong" kind of noise from the detector. + +

Error Handling

+Problems in ECal should be automatically corrected within one minute.
+For other subsystems, check if there is noise in the reported channels, discuss with detector experts. +

If the problem does not vanish, try a reboot of the sub-system with the next DAQ restart. Few errors are acceptable and don't need immediate action. + diff --git a/hmon/hmon_cosmic_alarms.pl b/hmon/hmon_cosmic_alarms.pl deleted file mode 100644 index 57776cb..0000000 --- a/hmon/hmon_cosmic_alarms.pl +++ /dev/null @@ -1,208 +0,0 @@ -#!/usr/bin/perl - -use warnings; -use strict; -use Data::Dumper; -use HADES::TrbNet; -use List::Util qw[min max]; -use Email::Sender::Simple qw(sendmail); -use Email::Simple; -use Email::Simple::Creator; - -use Log::Log4perl qw(get_logger); - -################################ -# To disable script: just uncomment -# the following line with "exit" -################################ -exit; -################################ - - -my $HADESDAQ="/home/hadaq/trbsoft/hadesdaq/"; - -Log::Log4perl::init($HADESDAQ . "hmon/automatic_restart_logger.conf"); -my $logger = get_logger("automatic_restart"); - -$logger->info("startup"); - -print STDERR "restarted. The log file is here: -/home/hadaq/trbsoft/hadesdaq/hmon/logs/automatic_restart.log -"; - -trb_init_ports() or die trb_strerror(); - -my $mail_fn = $HADESDAQ . "main/alarm_mail_list.txt"; - -my $number_of_reset_retries = 4; - -my $num_of_entries = 10; # loop 10 times and build average - -# number of MDC OEPs which are allowed to be misssing before alarm -my $max_allowed_mdc_oeps_missing = 1; - -# minimal average DAQ rate allowed before an alarm -my $lower_rate_limit = 150; -my $upper_rate_limit = 450; - - -my $beep_intervall = 10; -my $mail_intervall = 600; - -my $rh_data = {}; - -my $fh; - -open ($fh, "<", "/tmp/mdc_number_missing_boards") || die "could not open /tmp/mdc_number_missing_boards"; -$fh->autoflush(1); - - -my $time_last_beep_alarm=0; -my $time_last_mail_alarm=0; - -my $CURRENT_STATE = "idle"; -my $NEXT_STATE = "idle"; -my $reset_counter = 0; -my $iterator = 0; -while (1) { - - STATE_SWITCH: - for ($CURRENT_STATE) { - if (/idle/) { - last STATE_SWITCH; - } - if (/error/) { - last STATE_SWITCH; - } - if (/reset/) { - if ($reset_counter < $number_of_reset_retries - && $iterator >= $num_of_entries) { - $reset_counter++; - $logger->debug("doing a reset now: number of resets: $reset_counter"); - my $c = q|echo "

Doing an automatic TrbNet reset
" > ~/trbsoft/hadesdaq/hmon/files/note.htt; ssh lxhadesdaqp 'cd ~/trbsoft/daq/main; ./startup_briccolage.sh dont_restart_monitoring'; rm ~/trbsoft/hadesdaq/hmon/files/note.htt|; - #print STDERR localtime() . ": command: $c\n"; - qx($c); - $c=q|ssh lxhadesdaqp '/usr/bin/wget -a /tmp/EB_filestart.log -O /tmp/EB_fileres.txt "http://lxhadeb12:8099/Master/BNET/StartRun/execute?prefix=co&oninit=10"'|; - #print STDERR localtime() . ": command: $c\n"; - qx($c); - $iterator = 0; - @{$rh_data->{'rates'}} =(); - last STATE_SWITCH; - } - } - } - - my $daq_rate; - - my $rh_rate = trb_register_read(0x0003, 0xa001) or sleep 5 and next; - - if ( !defined $rh_rate) { - $daq_rate = 0; - } - else { - $daq_rate = $rh_rate->{0x0003} & 0xfffff; # only 20bits - } - - #print STDERR "daq_rate: $daq_rate\n"; - push(@{$rh_data->{'rates'}}, $daq_rate); - $iterator++; - my $num_datapoints = scalar @{$rh_data->{'rates'}}; - #print STDERR "iterator: $iterator: num_data_points: $num_datapoints\n"; - if ($iterator >= $num_of_entries) { - shift @{$rh_data->{'rates'}}; - } - - my $sum = 0; - foreach (@{$rh_data->{'rates'}}) { - $sum += $_; - } - my $average_rate = $sum / $num_of_entries; - #print STDERR "average_rate: $average_rate\n"; - seek($fh, 0, 0); - my $num_mdc_missing = <$fh>; - chomp $num_mdc_missing; - - my $state_string = "num missing: $num_mdc_missing , average_rate: $average_rate"; - #print "state_string: $state\n"; - - if($iterator >= $num_of_entries ) { - # ($average_rate > $upper_rate_limit) || - if ( ($average_rate < $lower_rate_limit) || - ($num_mdc_missing > $max_allowed_mdc_oeps_missing) ) { - my $time = time(); - $logger->debug("going to state in_error: average_rate: $average_rate, mdc_missing: $num_mdc_missing, iterator: $iterator\n"); - $NEXT_STATE = "reset"; - if ($time - $time_last_beep_alarm > $beep_intervall) { - $time_last_beep_alarm=$time; - $logger->debug("beep: state: $state_string"); - my $c; - $c = q|ssh -X hadesp30 'DISPLAY=:1 xterm -iconic -e bash -c "~/bin/bell;"' &|; - print "command: $c\n"; - qx($c); - - $c = "mpv --audio-device=auto ~/Documents/foghorn-daniel_simon.mp3 >/dev/null 2>/dev/null $mail_intervall && $reset_counter >= $number_of_reset_retries) { - - $time_last_mail_alarm = $time; - - my $mail_fh; - open ($mail_fh, "<", "$mail_fn") || print "could not open file: $mail_fn\n"; - - my @mails = <$mail_fh>; - $mail_fh->close; - chomp @mails; - foreach my $cur_mail (@mails) { - - next if($cur_mail=~/^#/ || $cur_mail=~/^\s+/ || $cur_mail=~/^\n/); - next unless $cur_mail=~/\w+@\w+/; - $logger->debug("current mail sent to: $cur_mail" ); - my $reset_counter_str = $reset_counter; - my $email = Email::Simple->create( - header => [ - To => "$cur_mail", - From => '"HADES DAQ" ', - Subject => 'Alarm during cosmic data-taking', - ], - body => "Error Condition: -- MDC-missing-OEPs: $num_mdc_missing -- average DAQ-rate: $average_rate - - lower_limit: $lower_rate_limit: upper_limit: $upper_rate_limit - - if it is outside the limits it might be due to some issue in the RPC. -- number of resets already tried: $reset_counter_str -", - ); - - sendmail($email) or $logger->error("error sending mail to $cur_mail"); - - } # loop over mail addresses - - } # if mail intervall - - } # error condition - else { - if($CURRENT_STATE eq "reset") { - $NEXT_STATE = "idle"; - $reset_counter=0; - $iterator = 0; - @{$rh_data->{'rates'}} =(); - $logger->info("leaving error state after $reset_counter number of resets: state: $state_string"); - } - } - - - if($iterator%60==0) { - $logger->debug("still alive signal: $state_string, state: $CURRENT_STATE, iterator: $iterator"); - } - - } # only if iterator is large enough make error meesages. - - sleep 1; - $CURRENT_STATE = $NEXT_STATE; -} #endless loop - - - diff --git a/hmon/hmon_endpoints.pl b/hmon/hmon_endpoints.pl index e68f411..ae14682 100755 --- a/hmon/hmon_endpoints.pl +++ b/hmon/hmon_endpoints.pl @@ -385,7 +385,7 @@ while(1) { $qastate = QA::GetQAState('below',$num_rich_missing,@QA::RichEndpMissingLimits); - $qastate = QA::NOTE if $num_rich_missing > 900; +# $qastate = QA::NOTE if $num_rich_missing > 900; if($qastate > 60) { system("logger -p local1.info -t DAQ Endp \\ $rich_longtext") unless (($timecnt->{rich}++)%$loggerperiod); } diff --git a/hmon/hmon_fwrate.pl b/hmon/hmon_fwrate.pl index 2c19d76..25cc134 100755 --- a/hmon/hmon_fwrate.pl +++ b/hmon/hmon_fwrate.pl @@ -22,9 +22,9 @@ my @x = (0,16,17,18,19,20,21,22,23,24,25,26,27,16,17,18,19,20,21,22,23,24,25,26, my @y = (0, 16,16,16,16,16,16,16,16,16,16,16,16,17,17,17,17,17,17,17,17,17,17,17,17,18,18,18,18,18,18,18,18,18,18,18,18,19,19,19,19,19,19,19,19,19,19,19,19,20,20,20,20,20,20,20,20,20,20,20,20,21,21,21,21,21,21,21,21,21,21,21,21,22,22,22,22,22,22,22,22,22,22,22,22,23,23,23,23,23,23,23,23,23,23,23,23,24,24,24,24,24,24,24,24,24,24,24,24,25,25,25,25,25,25,25,25,25,25,25,25,26,26,26,26,26,26,26,26,26,26,26,26,27,27,27,27,27,27,27,27,27,27,27,27,12,12,12,12,12,12,12,12,12,12,14,14,14,14,14,14,14,14,14,14,16,16,16,16,18,18,18,18,20,20,20,20,22,22,22,22,24,24,24,24,26,26,26,26,28,28,28,28,28,28,28,28,28,28,30,30,30,30,30,30,30,30,30,30,0,0,0,0,0,0,0,0,0,0,0,4,4,4,4,4,4,4,4,4,4,4,8,8,8,8,8,8,8,8,8,8,8,12,12,12,12,12,12,16,16,16,16,16,16,20,20,20,20,20,20,24,24,24,24,24,24,28,28,28,28,28,28,32,32,32,32,32,32,32,32,32,32,32,36,36,36,36,36,36,36,36,36,36,36,40,40,40,40,40,40,40,40,40); -my @bo = (0,0x6700, 0x6700, 0x6700, 0x6700, 0x6700, 0x6700, 0x6700, 0x6700, 0x6700, 0x6723, 0x6700, 0x6700, 0x6700, 0x6700, 0x6700, 0x6700, 0x6700, 0x6700, 0x6700, 0x6723, 0x6700, 0x6700, 0x6700, 0x6700, 0x6700, 0x6700, 0x6700, 0x6723, 0x6700, 0x6700, 0x6700, 0x6701, 0x6701, 0x6701, 0x6701, 0x6701, 0x6701, 0x6723, 0x6701, 0x6701, 0x6701, 0x6701, 0x6701, 0x6701, 0x6701, 0x6701, 0x6701, 0x6720, 0x6701, 0x6701, 0x6701, 0x6701, 0x6701, 0x6701, 0x6701, 0x6701, 0x6701, 0x6701, 0x6701, 0x6701, 0x6701, 0x6701, 0x6701, 0x6702, 0x0000, 0x0000, 0x6702, 0x6720, 0x6702, 0x6723, 0x6723, 0x6723, 0x6723, 0x6702, 0x6702, 0x6702, 0x0000, 0x0000, 0x6702, 0x6702, 0x6702, 0x6702, 0x6702, 0x6702, 0x6702, 0x6702, 0x6702, 0x6702, 0x6702, 0x6702, 0x6702, 0x6702, 0x6702, 0x6702, 0x6702, 0x6702, 0x6702, 0x6702, 0x6702, 0x6703, 0x6703, 0x6703, 0x6703, 0x6722, 0x6703, 0x6703, 0x6703, 0x6703, 0x6703, 0x6703, 0x6703, 0x6703, 0x6703, 0x6703, 0x6703, 0x6703, 0x6703, 0x6703, 0x6703, 0x6703, 0x6703, 0x6703, 0x6703, 0x6723, 0x6723, 0x6723, 0x6723, 0x6703, 0x6703, 0x6703, 0x6710, 0x6710, 0x6710, 0x6710, 0x6710, 0x6723, 0x6710, 0x6710, 0x6710, 0x6710, 0x6710, 0x6710, 0x6710, 0x6710, 0x6710, 0x6710, 0x6710, 0x6710, 0x6710, 0x6710, 0x6710, 0x6710, 0x6710, 0x6710, 0x6710, 0x6710, 0x6710, 0x6710, 0x6710, 0x6710, 0x6710, 0x6710, 0x6711, 0x6711, 0x6711, 0x6711, 0x6711, 0x6711, 0x6711, 0x6711, 0x6711, 0x6711, 0x6711, 0x6711, 0x6711, 0x6711, 0x6711, 0x6711, 0x6711, 0x6711, 0x6711, 0x6711, 0x6711, 0x6711, 0x6711, 0x6711, 0x6711, 0x6711, 0x6711, 0x6711, 0x6711, 0x6711, 0x6711, 0x6712, 0x6712, 0x6712, 0x6712, 0x6723, 0x6712, 0x6712, 0x6712, 0x6712, 0x6712, 0x6712, 0x6712, 0x6712, 0x6712, 0x0000, 0x0000, 0x6712, 0x6712, 0x6712, 0x6712, 0x6712, 0x6712, 0x6712, 0x0000, 0x0000, 0x0000, 0x6712, 0x6712, 0x6712, 0x6712, 0x6712, 0x6712, 0x6712, 0x6712, 0x6712, 0x0000, 0x6712, 0x6712, 0x6713, 0x6713, 0x6713, 0x6713, 0x6713, 0x6713, 0x6713, 0x6713, 0x6713, 0x6713, 0x6713, 0x6713, 0x6713, 0x6713, 0x6713, 0x6713, 0x6713, 0x6713, 0x6722, 0x6722, 0x6722, 0x6722, 0x6722, 0x6722, 0x6722, 0x6722, 0x6720, 0x6720, 0x6720, 0x6720, 0x6720, 0x6720, 0x6720, 0x6720, 0x6720, 0x6720, 0x6720, 0x6720, 0x6720, 0x6720, 0x6720, 0x6720, 0x6720, 0x6720, 0x6720, 0x6720, 0x6720, 0x6720, 0x6720, 0x6720, 0x0000, 0x6722, 0x6722, 0x6722, 0x6722, 0x6722, 0x6722, 0x6722, 0x6722, 0x6722, 0x0000, 0x0000, 0x0000, 0x6722, 0x6722, 0x6722, 0x6722, 0x6722, 0x6722, 0x6722, 0x0000, 0x0000); +my @bo = (0,0x6700,0x6700,0x6700,0x6700,0x6700,0x6700,0x6700,0x6700,0x6700,0x6700,0x6723,0x6700,0x6700,0x6700,0x6700,0x6700,0x6700,0x6700,0x6700,0x6700,0x6723,0x6700,0x6700,0x6700,0x6700,0x6700,0x6700,0x6700,0x6723,0x6700,0x6700,0x6700,0x6701,0x6701,0x6701,0x6701,0x6701,0x6701,0x6723,0x6701,0x6701,0x6701,0x6701,0x6701,0x6701,0x6701,0x6701,0x6701,0x6720,0x6701,0x6701,0x6701,0x6701,0x6701,0x6701,0x6701,0x6701,0x6701,0x6701,0x6701,0x6701,0x6701,0x6701,0x6701,0x6702,0x0000,0x0000,0x6702,0x6720,0x6702,0x6723,0x6723,0x6723,0x6723,0x6702,0x6702,0x6702,0x0000,0x0000,0x6702,0x6702,0x6702,0x6702,0x6702,0x6702,0x6702,0x6702,0x6702,0x6702,0x6702,0x6702,0x6702,0x6702,0x6702,0x6702,0x6702,0x6702,0x6702,0x6702,0x6702,0x6703,0x6703,0x6703,0x6703,0x6722,0x6703,0x6703,0x6703,0x6703,0x6703,0x6703,0x6703,0x6703,0x6703,0x6703,0x6703,0x6703,0x6703,0x6703,0x6703,0x6703,0x6703,0x6703,0x6703,0x6723,0x6723,0x6723,0x6723,0x6703,0x6703,0x6703,0x6710,0x6710,0x6710,0x6710,0x6710,0x6723,0x6710,0x6710,0x6710,0x6710,0x6710,0x6710,0x6710,0x6710,0x6710,0x6710,0x6710,0x6710,0x6710,0x6710,0x6710,0x6710,0x6710,0x6710,0x6710,0x6710,0x6710,0x6710,0x6710,0x6710,0x6710,0x6710,0x6711,0x6711,0x6711,0x6711,0x6711,0x6711,0x6711,0x6711,0x6711,0x6711,0x6711,0x6711,0x6711,0x6711,0x6711,0x6711,0x6711,0x6711,0x6711,0x6711,0x6711,0x6711,0x6711,0x6711,0x6711,0x6711,0x6711,0x6711,0x6711,0x6711,0x6711,0x6712,0x6712,0x6712,0x6712,0x6723,0x6712,0x6712,0x6712,0x6712,0x6712,0x6712,0x6712,0x6712,0x6712,0x0000,0x0000,0x6712,0x6712,0x6712,0x6712,0x6712,0x6712,0x6712,0x0000,0x0000,0x0000,0x6712,0x6712,0x6712,0x6712,0x6712,0x6712,0x6712,0x6712,0x6712,0x0000,0x6712,0x6712,0x6713,0x6713,0x6713,0x6713,0x6713,0x6713,0x6713,0x6713,0x6713,0x6713,0x6713,0x6713,0x6713,0x6713,0x6713,0x6713,0x6713,0x6713,0x6722,0x6722,0x6722,0x6722,0x6722,0x6722,0x6722,0x6722,0x6720,0x6720,0x6720,0x6720,0x6720,0x6720,0x6720,0x6720,0x6720,0x6720,0x6720,0x6720,0x6720,0x6720,0x6720,0x6720,0x6720,0x6720,0x6720,0x6720,0x6720,0x6720,0x6720,0x6720,0x0000,0x6722,0x6722,0x6722,0x6722,0x6722,0x6722,0x6722,0x6722,0x6722,0x0000,0x0000,0x0000,0x6722,0x6722,0x6722,0x6722,0x6722,0x6722,0x6722,0x0000,0x0000); -my @ch = (0,2,3,4,5,6,7,8,9,10,26,12,13,14,15,16,17,18,19,20,30,22,23,24,25,26,27,28,28,30,31,32,1,2,3,4,5,6,27,8,9,10,11,12,13,14,15,16,24,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,1,0,0,2,31,4,29,13,14,15,9,10,11,0,0,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,1,2,3,4,16,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,9,10,11,12,30,31,32,1,2,3,4,5,31,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,20,21,22,23,24,25,26,27,28,29,30,31,32,1,2,3,4,32,6,7,8,9,10,11,12,13,14,0,0,15,16,17,18,19,20,21,0,0,0,22,23,24,25,26,27,28,29,30,0,31,32,1,2,3,4,5,6,8,10,11,13,14,15,17,18,19,20,21,22,17,18,19,20,21,22,23,24,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,32,0,1,2,3,4,5,6,7,8,9,0,0,0,10,11,12,13,14,15,32,0,0); +my @ch = (0,1,2,3,4,5,6,7,8,9,10,26,12,13,14,15,16,17,18,19,20,30,22,23,24,25,26,27,28,28,30,31,32,1,2,3,4,5,6,27,8,9,10,11,12,13,14,15,16,24,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,1,0,0,2,31,4,29,13,14,15,9,10,11,0,0,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,1,2,3,4,16,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,9,10,11,12,30,31,32,1,2,3,4,5,31,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,20,21,22,23,24,25,26,27,28,29,30,31,32,1,2,3,4,32,6,7,8,9,10,11,12,13,14,0,0,15,16,17,18,19,20,21,0,0,0,22,23,24,25,26,27,28,29,30,0,31,32,1,2,3,4,5,6,8,10,11,13,14,15,17,18,19,20,21,22,17,18,19,20,21,22,23,24,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,32,0,1,2,3,4,5,6,7,8,9,0,0,0,10,11,12,13,14,15,32,0,0); HPlot::PlotInit({ name => "FwRate", @@ -119,15 +119,15 @@ while(1) { my $o = trb_register_read_mem(0xfe47,0xdfc0,0,32) or die trb_strerror() or sleep 5 and next; if (defined $old) { - my $tdiff = (time() - $oldtime)|1; + my $tdiff = (time() - $oldtime)||1; foreach my $b (keys %$o) { #my $ratesum = 0; next unless ($b>=0x6700 && $b<=0x67ff); for my $v (0..31) { - my $vdiff = ($o->{$b}->[$v]&0xfffffff) - ($old->{$b}->[$v]&0xfffffff); + my $vdiff = ($o->{$b}->[$v]&0xffffff) - ($old->{$b}->[$v]&0xffffff); - if ($vdiff < 0) { $vdiff += 2**28;} + if ($vdiff < 0) { $vdiff += 2**24;} $diff->{$b}->[$v] = $vdiff/$tdiff; HPlot::PlotFill('FwRateSimple',$diff->{$b}->[$v],$v,($b%4)+((($b>>4)%4)*4-($b>=0x6722?1:0))); diff --git a/hmon/hmon_ipcheck.pl b/hmon/hmon_ipcheck.pl old mode 100755 new mode 100644 index c186fd3..5176371 --- a/hmon/hmon_ipcheck.pl +++ b/hmon/hmon_ipcheck.pl @@ -81,7 +81,7 @@ while(1) { if($err) { $errorcnt++; $longstatus .= "Hub $b ($det) not reached
"; -# print $longstatus."\n"; + print $longstatus."\n"; $qastate = QA::ERROR; } } diff --git a/hmon/hmon_itofrate.pl b/hmon/hmon_itofrate.pl index 08f7610..0ec848f 100644 --- a/hmon/hmon_itofrate.pl +++ b/hmon/hmon_itofrate.pl @@ -92,7 +92,7 @@ while(1) { #my $min = 5E9; my $o = trb_register_read_mem(0xfe74,0xdfc0,0,48) or die trb_strerror() or sleep 5 and next; - my $tdiff = (time() - $oldtime)|1; + my $tdiff = (time() - $oldtime)||1; if (defined $old) { foreach my $board (keys %$o) { diff --git a/hmon/hmon_lvl1mismatch_autorestart.pl b/hmon/hmon_lvl1mismatch_autorestart.pl index 9014903..ccd1eb7 100755 --- a/hmon/hmon_lvl1mismatch_autorestart.pl +++ b/hmon/hmon_lvl1mismatch_autorestart.pl @@ -42,6 +42,7 @@ while(1){ $cnt--; $msg .= "->AR"; $cnt2++; + system("logger -p local1.info -t DAQ 'RICH Re-Included DiRich to DAQ: $msg'"); } } } diff --git a/hmon/hmon_richrate.pl b/hmon/hmon_richrate.pl index c65cd19..96fcdfd 100755 --- a/hmon/hmon_richrate.pl +++ b/hmon/hmon_richrate.pl @@ -141,7 +141,7 @@ while(1) { my $o = trb_register_read_mem(0xfe51,0xc000,0,33) or die trb_strerror() or sleep 5 and next; if (defined $old) { - my $tdiff = (time() - $oldtime)|1; + my $tdiff = (time() - $oldtime)||1; foreach my $b (keys %$o) { #my $ratesum = 0; my $sec = ($b>>1) & 0x3; diff --git a/hmon/hmon_rpcrate.pl b/hmon/hmon_rpcrate.pl index 6564092..1f1be21 100755 --- a/hmon/hmon_rpcrate.pl +++ b/hmon/hmon_rpcrate.pl @@ -76,7 +76,7 @@ while(1) { my $o = trb_register_read_mem(0xfe73,0xdfc0,0,48) or die trb_strerror() or sleep 5 and next; if (defined $old) { - my $tdiff = (time() - $oldtime)|1; + my $tdiff = (time() - $oldtime)||1; foreach my $b (keys %$o) { #my $ratesum = 0; next unless ($b>=0x5800 && $b<=0x58ff); diff --git a/hmon/hmon_rpctof_sectorwise_highres.pl b/hmon/hmon_rpctof_sectorwise_highres.pl index 033b495..fa2a051 100755 --- a/hmon/hmon_rpctof_sectorwise_highres.pl +++ b/hmon/hmon_rpctof_sectorwise_highres.pl @@ -23,13 +23,14 @@ HPlot::PlotInit({ sizex => 950, sizey => 330, xmin => 0, - curvewidth => .9, - xmax => 2000, + ymin => 0, + curvewidth => .9, + xmax => 2000, # nokey => 1, buffer => 1, - storable=> 1 +# storable=> 1 } - ); + ); my $str = Hmon::MakeTitle(12,7,"RpcTofSectorRate_highres",0); $str .= qq@@; @@ -49,7 +50,7 @@ my $diff; while (1) { $iter++; - + my $rpc = trb_registertime_read(0xfe61,0xdfe6); my $tof = trb_registertime_read(0xfe40,0xdfd3); @@ -62,7 +63,9 @@ while (1) { $vdiff += 2**24 if ($vdiff < 0); $diff->{$b} = $vdiff/$tdiff; if ($b >= 0x84c0 && $b <= 0x84c5) { + #print Dumper $diff->{$b}, $b&0xF; HPlot::PlotAdd( 'RpcTofSectorRateHist',$diff->{$b},$b&0xF); + #HPlot::PlotAdd( 'RpcTofSectorRateHist', 100); } } foreach my $b (keys %$tof) { @@ -73,6 +76,7 @@ while (1) { $vdiff += 2**24 if ($vdiff < 0); $diff->{$b} = $vdiff/$tdiff; if ($b >= 0x86c0 && $b <= 0x86c5) { + #print Dumper $diff->{$b}, ($b&0xF)+6; HPlot::PlotAdd( 'RpcTofSectorRateHist',$diff->{$b},($b&0xF)+6); } } @@ -82,6 +86,7 @@ while (1) { $oldtof = $tof; HPlot::PlotDrawFork('RpcTofSectorRateHist') if $iter == 10; + #HPlot::PlotDrawFork('RpcTofSectorRateHist'); $iter = 0 if $iter == 10; usleep(100000); } diff --git a/hmon/hmon_seu.pl b/hmon/hmon_seu.pl index 9365735..5b24f07 100755 --- a/hmon/hmon_seu.pl +++ b/hmon/hmon_seu.pl @@ -57,7 +57,8 @@ while(1) { $longtext = "$seuboards out of $totalboards monitored report some radiation issues
"; $longtext .= sprintf("%04x (%s)
",$_,Hmon::getsystem($_)) for sort @boards; - if($seuboards >= 12) { + if($seuboards >= 8) { + $longtext .= "
Quite many boards seem to be affected by radiation. It's suggested to reboot \"Quite-a-lot\" before the next DAQ restart."; $qastate = QA::WARN; } if($seuboards >= 16) { diff --git a/hmon/hmon_spill.pl b/hmon/hmon_spill.pl index f92fac2..e7b3291 100755 --- a/hmon/hmon_spill.pl +++ b/hmon/hmon_spill.pl @@ -106,7 +106,6 @@ while (1) { my $qashort; my $qalong; - if (!$spillsignal && $lastspillsignal) { $spillcount++; @@ -135,7 +134,13 @@ while (1) { $qashort, $qalong) unless $opt_debug>0; HPlot::PlotAdd("EvtsPerSpill",$events/1E3) unless $opt_debug>0; HPlot::PlotDraw("EvtsPerSpill") unless $opt_debug>0; - $events = 0; + if(($events < $QA::LimitTriggerPerSpill[0] && $qastate > QA::OK)) { + my $prefix = get ($url_prefix); + Hmon::Speak('nobeam', "No beam") if ($prefix =~ 'be'); + } + + + $events = 0; $spilllength = 0; } elsif ($spilllength > 30) { QA::WriteQALog($fqa, "trg", "spill", 30, QA::NOTE, "Spill Sum", @@ -182,7 +187,7 @@ if ($iter%2) { $qashort, $qalong) unless $opt_debug>0; } my $spillcountstate = QA::OK; - if ($countnochange > 50) { + if ($countnochange > $QA::AcceleratorCycle * 4) { #double normal duration $spillcountstate = QA::WARN; # my $prefix = get ($url_prefix); # $prefix = "--" unless defined $prefix; diff --git a/hmon/hmon_startratesimple.pl b/hmon/hmon_startratesimple.pl index b3e2bf6..8c9837a 100755 --- a/hmon/hmon_startratesimple.pl +++ b/hmon/hmon_startratesimple.pl @@ -98,7 +98,7 @@ while (1) { #print Dumper $o; if (defined $old) { - my $tdiff = (time() - $oldtime)|1; + my $tdiff = (time() - $oldtime)||1; foreach my $endpoint (sort keys %$current) { #foreach my $endpoint (0x5000, 0x5001, 0x5002, 0x5003, 0x5004) { #my $ratesum = 0; diff --git a/hmon/hmon_timeout.pl b/hmon/hmon_timeout.pl index 15dcbca..58edfbf 100755 --- a/hmon/hmon_timeout.pl +++ b/hmon/hmon_timeout.pl @@ -71,11 +71,11 @@ while (1) { $qastate = QA::GetQAState('below', $cnt+$cnt2, @QA::TimeoutLimitsOnlyRICHMDC); } - if($boardlist1 ne "") { + if($boardlist1 ne "" && $qastate > QA::NOTE) { Hmon::Speak('timeout'.(($qastate= 3; } - if($boardlist2 ne "" && $boardlist1 eq "" ) { + if($boardlist2 ne "" && $boardlist1 eq "" && $qastate > QA::NOTE) { Hmon::Speak("timeout".(($qastate= 3; } diff --git a/hmon/hmon_tofrate.pl b/hmon/hmon_tofrate.pl index 980493f..188390f 100755 --- a/hmon/hmon_tofrate.pl +++ b/hmon/hmon_tofrate.pl @@ -75,7 +75,7 @@ while(1) { my $o = trb_register_read_mem(0xfe47,0xdfc0,0,32) or die trb_strerror() or sleep 5 and next; if (defined $old) { - my $tdiff = (time() - $oldtime)|1; + my $tdiff = (time() - $oldtime)||1; foreach my $b (keys %$o) { #my $ratesum = 0; next unless ($b>=0x5c00 && $b<=0x5cff); diff --git a/hmon/hmon_veto.pl b/hmon/hmon_veto.pl index ba206b4..97c0b91 100755 --- a/hmon/hmon_veto.pl +++ b/hmon/hmon_veto.pl @@ -40,12 +40,44 @@ HPlot::PlotInit({ showvalues => 0, }); +HPlot::PlotInit({ + name => "Veto_histbar", + file => "files/Veto_histbar", + entries => 200, + type => HPlot::TYPE_BARGRAPH, + output => HPlot::OUT_PNG, +# titles => ["0","1","2","3","4","5","6","7","8","9","10","11"], + xlabel => "Time / 1 s", + ylabel => "Hitrate [Hz]", + sizex => 950, + sizey => 330, + xmin => 0, + xmax => 200, + ymin => "0", + ymax => "100<*", + curves => 8, +# xscale => 10, # xscale does not work with TYPE_BARGRAPH + buffer => 1, + stacked => 1, + curvewidth => .9, +# additional => " +# set obj 1 rect from -1, 7E7 to 2000, 10E7 fc rgb '#ffffbb' behind \n +# set obj 2 rect from -1, 10E7 to 2000, 100E7 fc rgb '#ffdddd' behind +# " + }); + my $str = Hmon::MakeTitle(4, 6, "Veto Hit Rate",0); $str .= qq@
\n@; $str .= Hmon::MakeFooter(); Hmon::WriteFile("VetoMap",$str); +$str = Hmon::MakeTitle(12, 7, "Veto Summed Rates",0); +$str .= qq@
\n@; +$str .= Hmon::MakeFooter(); +Hmon::WriteFile("Veto_histbar",$str); + + my $old; my $oldtime = time(); my $time = time(); @@ -53,12 +85,13 @@ my $diff; my $iter = 0; +my $flog = QA::OpenQAFile(); trb_init_ports() or die trb_strerror(); while(1) { - + my $vetosum = 0; my $o = trb_register_read_mem(0x5004,0xc000,0,9) or die trb_strerror() or sleep 5 and next; @@ -72,7 +105,10 @@ while(1) { if ($vdiff < 0) { $vdiff += 2**24;} my $d = $vdiff/$tdiff; - + + HPlot::PlotAdd('Veto_histbar',$d,$v); + $vetosum += $d; + if($v == 1) { HPlot::PlotFill('VetoMap',$d,3,0); HPlot::PlotFill('VetoMap',$d,3,1); @@ -121,12 +157,14 @@ while(1) { } } HPlot::PlotDrawFork('VetoMap'); - - + HPlot::PlotDrawFork('Veto_histbar'); - } + my $vetomsg = QA::SciNotation($vetosum); + my $vetolongmsg = sprintf("counts per second %s/s", QA::SciNotation($vetosum)); + QA::WriteQALog($flog, "rate", "veto", 30, QA::OK, "Veto", $vetomsg, $vetolongmsg); + } $old = $o; $oldtime = time(); - sleep 1; + usleep(1000000); } diff --git a/hmon/hmon_waterfall.pl b/hmon/hmon_waterfall.pl index 1ccd587..6b7f944 100755 --- a/hmon/hmon_waterfall.pl +++ b/hmon/hmon_waterfall.pl @@ -47,56 +47,58 @@ my $str = Hmon::MakeTitle(12, 10, "Start Falls"); trb_init_ports() or die trb_strerror(); + my $channels_x_1 = [ - { pch => 1, tdc => 2, ch => 0}, # Int(ch/2) because we read from monitoring register, which counts 0-23 for each TDC. - { pch => 3, tdc => 2, ch => 1}, # Take care that the functions copy pasted DO NOT have a "-1" for the channels! - { pch => 5, tdc => 2, ch => 1}, # pch: physical channel. The thing which will be plotted - { pch => 7, tdc => 2, ch => 3}, # tdc: tdc number - { pch => 9, tdc => 2, ch => 4}, # ch: channel in the TDC - { pch => 11, tdc => 2, ch => 6}, - { pch => 13, tdc => 2, ch => 6}, - { pch => 15, tdc => 2, ch => 7} + { pch => 2, tdc => 2, ch => 7}, # Int(ch/2) because we read from monitoring register, which counts 0-23 for each TDC. + { pch => 4, tdc => 2, ch => 6}, # Take care that the functions copy pasted DO NOT have a "-1" for the channels! + { pch => 6, tdc => 2, ch => 5}, # pch: physical channel. The thing which will be plotted + { pch => 8, tdc => 2, ch => 4}, # tdc: tdc number + { pch => 10, tdc => 2, ch => 3}, # ch: channel in the TDC + { pch => 12, tdc => 2, ch => 2}, + { pch => 14, tdc => 2, ch => 1}, + { pch => 16, tdc => 2, ch => 0} ]; my $channels_x_0 = [ - { pch => 2 , tdc => 0, ch => 7 }, # Int(ch/2) because we read from monitoring register, which counts 0-23 for each TDC. - { pch => 4 , tdc => 0, ch => 6 }, # Take care that the functions copy pasted DO NOT have a "-1" for the channels! - { pch => 6 , tdc => 0, ch => 6 }, - { pch => 8 , tdc => 0, ch => 4 }, - { pch => 10, tdc => 0, ch => 3 }, - { pch => 12, tdc => 0, ch => 1 }, - { pch => 14, tdc => 0, ch => 1 }, - { pch => 16, tdc => 0, ch => 0 } + { pch => 1 , tdc => 0, ch => 0 }, # Int(ch/2) because we read from monitoring register, which counts 0-23 for each TDC. + { pch => 3 , tdc => 0, ch => 1 }, # Take care that the functions copy pasted DO NOT have a "-1" for the channels! + { pch => 5 , tdc => 0, ch => 2 }, + { pch => 7 , tdc => 0, ch => 3 }, + { pch => 9, tdc => 0, ch => 4 }, + { pch => 11, tdc => 0, ch => 5 }, + { pch => 13, tdc => 0, ch => 6 }, + { pch => 15, tdc => 0, ch => 7 } ]; my $channels_y_1 = [ - { pch => 2 , tdc => 3, ch => 7 }, # Int(ch/2) because we read from monitoring register, which counts 0-23 for each TDC. - { pch => 4 , tdc => 3, ch => 6 }, # Take care that the functions copy pasted DO NOT have a "-1" for the channels! - { pch => 6 , tdc => 3, ch => 6 }, - { pch => 8 , tdc => 3, ch => 4 }, - { pch => 10, tdc => 3, ch => 3 }, - { pch => 12, tdc => 3, ch => 1 }, - { pch => 14, tdc => 3, ch => 1 }, - { pch => 16, tdc => 3, ch => 0 } + { pch => 1 , tdc => 3, ch => 0 }, # Int(ch/2) because we read from monitoring register, which counts 0-23 for each TDC. + { pch => 3 , tdc => 3, ch => 1 }, # Take care that the functions copy pasted DO NOT have a "-1" for the channels! + { pch => 5 , tdc => 3, ch => 2 }, + { pch => 7 , tdc => 3, ch => 3 }, + { pch => 9, tdc => 3, ch => 4 }, + { pch => 11, tdc => 3, ch => 5 }, + { pch => 13, tdc => 3, ch => 6 }, + { pch => 15, tdc => 3, ch => 7 } ]; my $channels_y_0 = [ - { pch => 1, tdc => 1, ch => 0}, # Int(ch/2) because we read from monitoring register, which counts 0-23 for each TDC. - { pch => 3, tdc => 1, ch => 1}, # Take care that the functions copy pasted DO NOT have a "-1" for the channels! - { pch => 5, tdc => 1, ch => 1}, - { pch => 7, tdc => 1, ch => 3}, - { pch => 9, tdc => 1, ch => 4}, - { pch => 11, tdc => 1, ch => 6}, - { pch => 13, tdc => 1, ch => 6}, - { pch => 15, tdc => 1, ch => 7} + { pch => 2, tdc => 1, ch => 7}, # Int(ch/2) because we read from monitoring register, which counts 0-23 for each TDC. + { pch => 4, tdc => 1, ch => 6}, # Take care that the functions copy pasted DO NOT have a "-1" for the channels! + { pch => 6, tdc => 1, ch => 5}, + { pch => 8, tdc => 1, ch => 4}, + { pch => 10, tdc => 1, ch => 3}, + { pch => 12, tdc => 1, ch => 2}, + { pch => 14, tdc => 1, ch => 1}, + { pch => 16, tdc => 1, ch => 0} ]; + while (1) { my $st = trb_registertime_read_mem(0xfe58,0xc001,0,8); diff --git a/hmon/index.cgi b/hmon/index.cgi index 7bbe153..88bdf40 100755 --- a/hmon/index.cgi +++ b/hmon/index.cgi @@ -35,7 +35,7 @@ function openwin(url) {
  • Web Tools (daqtools)
  • Eventbuilder Monitor
  • Raw Data Monitor
    • -
  • Accumulated Run Statistics +
  • Accumulated Run Statistics

    Documents

    Operator Monitor - the "must-have" windows

    diff --git a/hmon/monitor.cgi b/hmon/monitor.cgi index 10ac4e4..dc3e3c0 100755 --- a/hmon/monitor.cgi +++ b/hmon/monitor.cgi @@ -221,6 +221,7 @@ setInterval("resize()",1000);*/ $; } $out .= qq$ +
    $; diff --git a/hmon/permanent/Perl2Epics2.pm b/hmon/permanent/Perl2Epics2.pm new file mode 120000 index 0000000..1e81d36 --- /dev/null +++ b/hmon/permanent/Perl2Epics2.pm @@ -0,0 +1 @@ +../Perl2Epics2.pm \ No newline at end of file diff --git a/hmon/permanent/hmon_beamabort.pl b/hmon/permanent/hmon_beamabort.pl index e039a89..9176692 100755 --- a/hmon/permanent/hmon_beamabort.pl +++ b/hmon/permanent/hmon_beamabort.pl @@ -66,8 +66,8 @@ while (1) { $longtext .= "none" if (substr($res[3],3,1) ne "1"); $longtext .= "ABORTED" if (substr($res[3],3,1) eq "1"); - $value = "ABORTED" if $longtext =~ /ABORTED/; $value = "ABORT" if $longtext =~ /ABORT/ && $value ne ""; + $value = "ABORTED" if $longtext =~ /ABORTED/; $value = "inactive" if $longtext =~ /inactive/; $qastate = QA::WARN if $longtext =~ /inactive/; $qastate = QA::FATAL if $longtext =~ /ABORT/; @@ -110,6 +110,7 @@ while (1) { if($qastate >= QA::ERROR && $laststate == QA::OK) { system("ssh hadaq\@hadesp33 'cd /home/hadaq/trbsoft/daq/hmon; PERL5LIB=. perl ./permanent/hmon_archiver.pl once'"); + system("logger -p local1.info -t DAQ Abort \\ Beam abort triggered"); Hmon::Speak('beamabort',"Beam abort. Beam abort."); } diff --git a/hmon/permanent/hmon_eb_missing_data.pl b/hmon/permanent/hmon_eb_missing_data.pl index ceb9d2c..da1c561 100755 --- a/hmon/permanent/hmon_eb_missing_data.pl +++ b/hmon/permanent/hmon_eb_missing_data.pl @@ -24,6 +24,7 @@ my @state; # with this switch disable qalog and enable printout only JAM my $opt_debug = 0; my $speak_counter =0; +my $noData_counter =0; my $fqa = QA::OpenQAFile(); @@ -161,33 +162,48 @@ my $qamsgshort = ""; } - - - if (index($masterstate, 'NoData') != -1) { - $qastate = QA::ERROR; + $noData_counter++; + if($noData_counter > 5) { + $qastate = QA::ERROR; + } + else { + $qastate = QA::NOTE; + } $qamsgshort = "All inputs get no data!"; - - $qamsg = "All eventbuilder inputs do not get any data. Check DAQ or accelerator!";# if ($opt_debug<1) { - + $qamsg = "All eventbuilder inputs do not get any data. Check DAQ or accelerator!"; } - if (index($masterstate, 'Mismatch') != -1) { + elsif (index($masterstate, 'no_data') != -1) { + $qastate = QA::ERROR; + my $missingin = substr($masterstate, rindex($masterstate,'_')+1); + $qamsgshort = "Input $missingin get no data!"; + $qamsg = "Eventbuilder input $missingin does not get any data. Try fix missing IP or EB reset."; + } + elsif (index($masterstate, 'blocked') != -1) { + $qastate = QA::ERROR; + my $blockedin = substr($masterstate, rindex($masterstate,'_')+1); + $qamsgshort = "Input $blockedin get no data!"; + $qamsg = "Eventbuilder input $blockedin is blocked. Try EB reset."; + } + elsif (index($masterstate, 'Mismatch') != -1) { $qamsgshort = "Server mismatch!"; $qastate = QA::ERROR; $qamsg .= "An eventbuilder server node is missing! Please restart eventbuilders."; - - } + } + else { + $noData_counter = 0; + } if ($opt_debug<1) { - $speak_counter++; - if($speak_counter>20) - { - $speak_counter=-20; - - Hmon::Speak('ebmissing',"Event builders are missing input data.") if $qastate > 60; - } + # $speak_counter++ if $qastate > 60; + # $speak_counter = 0 if $qastate < 60; + # if($speak_counter>10) + # { + # $speak_counter=-10; + Hmon::Speak('ebmissing',"Event builders are missing input data.") if $qastate > 60; + # } # dracones QA::WriteQALog($fqa, "server", "ebinputs", 10, $qastate, $qatitle, $qamsgshort, $qamsg); diff --git a/hmon/permanent/hmon_eb_run.pl b/hmon/permanent/hmon_eb_run.pl index c97533d..efdb918 100755 --- a/hmon/permanent/hmon_eb_run.pl +++ b/hmon/permanent/hmon_eb_run.pl @@ -429,15 +429,19 @@ while (1) { $evtavglong .= "
    Number of special triggers is not correct (debug $cnterrtype9 $cnterrtypeE)"; } if ($totalbytes < 20 && $lasttotalbytes <20) { - $qastate = QA::WARN_2; + $qastate = QA::NOTE; $speak_counter++; - if($speak_counter>20) + if($speak_counter>10) { - $speak_counter=-20; + $speak_counter=-10; + $qastate = QA::WARN_2; Hmon::Speak('ebfiles', "Event building not writing files to disk.") unless $opt_debug>0 ||($masterstate =~ /NoNodes/); print "No files are written by eventbuilders.\n" unless $opt_debug<1; } } + else { + $speak_counter = 0; + } if ($trbneterr == 0) { diff --git a/hmon/permanent/hmon_expertsoncall.pl b/hmon/permanent/hmon_expertsoncall.pl index 8c08ebc..c539168 100755 --- a/hmon/permanent/hmon_expertsoncall.pl +++ b/hmon/permanent/hmon_expertsoncall.pl @@ -47,25 +47,25 @@ while(1) { my $hour = strftime ("%H", localtime(time+$offset*86400+150)); $hour =~ s/\s//; - my $plan = get("https://hessenbox-a10.rz.uni-frankfurt.de/dl/fiUP5enanrB2pwwuBDJxGj/ExpertsPlan.csv"); - my @plan = split("\n",$plan); - - foreach my $line (@plan) { - my @s = split(",",$line); - if (scalar @s == 5 && $s[0] eq 'contact') { - $store->{contact}{$s[2]}{expert}=$s[1]; - $store->{contact}{$s[2]}{phone}=$s[3]; - $store->{contact}{$s[2]}{alias}=$s[4]; - $store->{alias}{$s[4]} = $s[2]; - $store->{alias}{$s[2]} = $s[4]; - } - if (scalar @s == 29 && $s[0] ne '') { - my $exp = $store->{contact}{$s[0]}{expert} //''; - if($nowmonth eq $s[2] && $nowday eq $s[3]) { - $store->{onshift}{$exp}{$s[0]} = $s[$hour+4]; - } - } - } +# my $plan = get("https://hessenbox-a10.rz.uni-frankfurt.de/dl/fiUP5enanrB2pwwuBDJxGj/ExpertsPlan.csv"); +# my @plan = split("\n",$plan); +# +# foreach my $line (@plan) { +# my @s = split(",",$line); +# if (scalar @s == 5 && $s[0] eq 'contact') { +# $store->{contact}{$s[2]}{expert}=$s[1]; +# $store->{contact}{$s[2]}{phone}=$s[3]; +# $store->{contact}{$s[2]}{alias}=$s[4]; +# $store->{alias}{$s[4]} = $s[2]; +# $store->{alias}{$s[2]} = $s[4]; +# } +# if (scalar @s == 29 && $s[0] ne '') { +# my $exp = $store->{contact}{$s[0]}{expert} //''; +# if($nowmonth eq $s[2] && $nowday eq $s[3]) { +# $store->{onshift}{$exp}{$s[0]} = $s[$hour+4]; +# } +# } +# } my $oraclecmd = "curl 'https://hades-db.gsi.de/pls/hades_webdbs/hades_oper.hshiftcrews2.show_shifts' -b /tmp/cookiefile -c /tmp/cookiefile -X POST -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' -H 'Accept-Language: de,en-US;q=0.7,en;q=0.3' -H 'Content-Type: application/x-www-form-urlencoded' -H 'Origin: https://hades-db.gsi.de' -H 'DNT: 1' -H 'Connection: keep-alive' -H 'Referer: https://hades-db.gsi.de/pls/hades_webdbs/hades_oper.hshiftcrews2.form_select' -H 'Upgrade-Insecure-Requests: 1' -H 'Sec-Fetch-Dest: iframe' -H 'Sec-Fetch-Mode: navigate' -H 'Sec-Fetch-Site: same-origin' -H 'Sec-Fetch-User: ?1' --data-raw 'p_exp_id=FEB24&p_begin=$oracledatestart&p_end=$oracledateend&p_purpose_id=&p_responsible=&p_action=Show+Shift+Crew' -u hades:6\\\$ectors 2>/dev/null" ; diff --git a/hmon/permanent/hmon_logbook.pl b/hmon/permanent/hmon_logbook.pl index 7825b2e..c0dcc45 100755 --- a/hmon/permanent/hmon_logbook.pl +++ b/hmon/permanent/hmon_logbook.pl @@ -15,7 +15,7 @@ use QA; my $firstrun =1; -my $command = "curl 'https://hades-db.gsi.de/pls/hades_webdbs/hades_oper.hlogbook2.show' -b /tmp/cookiefile -c /tmp/cookiefile -X POST -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' -H 'Accept-Language: de,en-US;q=0.7,en;q=0.3' -H 'Content-Type: application/x-www-form-urlencoded' -H 'Origin: https://hades-db.gsi.de' -H 'DNT: 1' -H 'Connection: keep-alive' -H 'Referer: https://hades-db.gsi.de/pls/hades_webdbs/hades_oper.hlogbook2.form_selection' -H 'Upgrade-Insecure-Requests: 1' -H 'Sec-Fetch-Dest: iframe' -H 'Sec-Fetch-Mode: navigate' -H 'Sec-Fetch-Site: same-origin' -H 'Sec-Fetch-User: ?1' --data-raw 'p_exp_id=FEB24&p_begin=&p_end=&p_num_days=&p_num_entries=5&p_author=&p_action=Show&p_log_type=All&p_run_type=None&p_current=&p_filename=*&p_search_text=*' -u hades:6\\\$ectors 2>/dev/null | iconv -f ISO-8859-1 -t UTF-8"; +my $command = "curl 'https://hades-db.gsi.de/pls/hades_webdbs/hades_oper.hlogbook2.show' -b /tmp/cookiefile -c /tmp/cookiefile -X POST -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' -H 'Accept-Language: de,en-US;q=0.7,en;q=0.3' -H 'Content-Type: application/x-www-form-urlencoded' -H 'Origin: https://hades-db.gsi.de' -H 'DNT: 1' -H 'Connection: keep-alive' -H 'Referer: https://hades-db.gsi.de/pls/hades_webdbs/hades_oper.hlogbook2.form_selection' -H 'Upgrade-Insecure-Requests: 1' -H 'Sec-Fetch-Dest: iframe' -H 'Sec-Fetch-Mode: navigate' -H 'Sec-Fetch-Site: same-origin' -H 'Sec-Fetch-User: ?1' --data-raw 'p_exp_id=APR24TEST&p_begin=&p_end=&p_num_days=&p_num_entries=5&p_author=&p_action=Show&p_log_type=All&p_run_type=None&p_current=&p_filename=*&p_search_text=*' -u hades:6\\\$ectors 2>/dev/null | iconv -f ISO-8859-1 -t UTF-8"; my $store; my $hash; diff --git a/hmon/permanent/hmon_magnet.pl b/hmon/permanent/hmon_magnet.pl index 2e10702..a2396c0 100755 --- a/hmon/permanent/hmon_magnet.pl +++ b/hmon/permanent/hmon_magnet.pl @@ -31,10 +31,10 @@ my %range_list = ( 'HAD:MAGNET:TC1238' => {},#{'min' => 4.6 , 'max' => 4.9 }, 'HAD:MAGNET:TC1248' => {},#{'min' => 4.5 , 'max' => 4.7 }, 'HAD:MAGNET:FI1400' => {},#{'min' => 0.36 , 'max' => 0.62 }, - 'HAD:MAGNET:PSUI_rbk' => {'min' => 1840 , 'max' => 1860 }, + 'HAD:MAGNET:MCS:PSUI_rbk' => {'min' => 1840 , 'max' => 1860 }, 'HAD:MAGNET:PSUV_rbk' => {},#{'min' => 0 , 'max' => 1 }, 'HAD:MAGNET:BBI' => {},#{'min' => 419 , 'max' => 620 }, - 'HAD:MAGNET:LL1708' => {'minerror' => 30, 'min' => 38, 'max' => 65, 'maxerror' => 65}, + 'HAD:MAGNET:MCS:LL1708' => {'minerror' => 30, 'min' => 38, 'max' => 65, 'maxerror' => 65}, 'HAD:MAGNET:TP1240' => {},#{'min' => 75.5 , 'max' => 89.0 , 'cal_max' => 2}, 'HAD:MAGNET:PI2610' => {},#{'min' => 1.7 , 'max' => 3.1 }, 'HAD:MAGNET:TP2200' => {},#{'min' => 83.0 , 'max' => 85.5 , 'cal_max' => 2}, @@ -134,7 +134,7 @@ while (1) { my $val = $epics->{$name}->{val}; $time = $epics->{$name}->{tme} if ($epics->{$name}->{tme} < $time) or ($time == 0); # print("$name $val\n"); - if ($name eq 'HAD:MAGNET:PSUI_rbk') { + if ($name eq 'HAD:MAGNET:MCS:PSUI_rbk') { $current = $val; # print("\t$name $val\n"); } @@ -146,7 +146,7 @@ while (1) { $TS105 = $val; # print("\t$name $val\n"); } - if ($name eq 'HAD:MAGNET:LL1708') { + if ($name eq 'HAD:MAGNET:MCS:LL1708') { $LL1708 = $val; # print("\t$name $val\n"); } @@ -208,15 +208,15 @@ while (1) { } #print $fatal_error_string . "\n"; if ($fatal_error_counter>3) { - $fatal_error_string .= " (last good check at $last_good_time)"; + $fatal_error_string .= "\n(last good check at $last_good_time, last check at $time)"; my $state = $magnet_on?(QA::FATAL):(QA::NOTE); QA::WriteQALog($flog,"misc","magnet",$interval+20,$state,$title,$value,$fatal_error_string); } elsif ($red_error_counter > 0) { - $fatal_error_string .= " (last good check at $last_good_time)"; + $fatal_error_string .= "\n(last good check at $last_good_time, last check at $time)"; my $state = $magnet_on?(QA::ERROR):(QA::NOTE); QA::WriteQALog($flog,"misc","magnet",$interval+20,$state,$title,$value,$fatal_error_string); } elsif ($yellow_error_counter > 0) { - $fatal_error_string .= " (last good check at $last_good_time)"; + $fatal_error_string .= "\n(last good check at $last_good_time, last check at $time)"; my $state = $magnet_on?(QA::WARN):(QA::NOTE); QA::WriteQALog($flog,"misc","magnet",$interval+20,$state,$title,$value,$fatal_error_string); } else { diff --git a/hmon/permanent/hmon_mdcpressure.pl b/hmon/permanent/hmon_mdcpressure.pl index f65dda9..6c0bc57 100755 --- a/hmon/permanent/hmon_mdcpressure.pl +++ b/hmon/permanent/hmon_mdcpressure.pl @@ -429,10 +429,10 @@ while(1) { $longmsg .= "VacHigh HI
    "; } if($qastat == QA::OK) { - $msg = sprintf("%s / %s",QA::SciNotation($vallow),QA::SciNotation($valhigh)); + $msg = sprintf("%s / %s",QA::SciNotation($vallow),QA::SciNotation2($valhigh)); } $longmsg .= sprintf("low vacuum: 20 mbar << 25 mbar < %s mbar < 80 mbar << 110 mbar
    ",QA::SciNotation($vallow)); - $longmsg .= sprintf("high vacuum: %s mbar < 60u mbar << 150u mbar
    ",QA::SciNotation($valhigh)); + $longmsg .= sprintf("high vacuum: %s mbar < 60×10-6 mbar << 150×10-6 mbar
    ",QA::SciNotation2($valhigh)); if($H2_run) { if ($data->{"Gate"}->{tme}==-1) { diff --git a/hmon/permanent/hmon_qa.pl b/hmon/permanent/hmon_qa.pl index 797353d..9fda6d1 100755 --- a/hmon/permanent/hmon_qa.pl +++ b/hmon/permanent/hmon_qa.pl @@ -74,6 +74,7 @@ while($a = ) { } } $str .="
    "; +# $str .= '
    '; $str .= Hmon::MakeFooter(); Hmon::WriteFile("QA",$str); diff --git a/hmon/permanent/hmon_richIsobutan.pl b/hmon/permanent/hmon_richIsobutan.pl index 355dcc2..a450bfb 100755 --- a/hmon/permanent/hmon_richIsobutan.pl +++ b/hmon/permanent/hmon_richIsobutan.pl @@ -101,6 +101,11 @@ HPlot::PlotInit($plotScale); my $timer = 0; my $offset_o2 = 0; my $str = ""; +#my @emailAddrs = ("juergen.friese\@ph.tum.de", "foertsch\@uni-wuppertal.de", "pauly\@physik.uni-wuppertal.de"); +my @emailAddrs = (); +my $mailoutWarn = 0; +my $mailoutErr = 0; +my $mailoutDaily = 0; my $s = "HAD:RICH:GAS:pipePressureIsoB"; Perl2Epics::Connect("IsoPress",$s); @@ -111,6 +116,8 @@ Perl2Epics::Connect("IsoScale1","HAD:scale:1:net"); Perl2Epics::Connect("IsoScale2","HAD:scale:2:net"); print "Connected.\n"; +usleep 1E6; +Perl2Epics::GetAll(); while(1) { my $data = Perl2Epics::GetAll(); @@ -173,6 +180,49 @@ while(1) { $qastate = min(QA::WARN_2,$qastate) if $QA::RichHvOff; $qastate = QA::NOTE if $QA::RichGasOff; + if($qastate == QA::WARN_2 && time()-$mailoutWarn > 60*60) { + foreach my $currEmailAddr (@emailAddrs) { + my $body = ""; + $body .= "The RICH gas system warns:\n"; + $body .= sprintf("Isobutan pressure : %.2f bar Warn(<0.9)\n",$IsobPress); + $body .= sprintf("O2 concentration : %.1f ppm Warn(>2000.0)\n",$IsobO2); + $body .= sprintf("Isobutanoutput ratio : %.2f Warn(<70.0)\n", $IsobRefR); + $body .= sprintf("Scales: %.1fkg / %.1fkg\n",,$IsobScale1,$IsobScale2); + Hmon::SendEmail($currEmailAddr,"GAS WARNING IN RICH",$body); + } + $mailoutWarn = time(); + } + + if($qastate == QA::ERROR_2 && time()-$mailoutErr > 60*60) { + foreach my $currEmailAddr (@emailAddrs) { + my $body = ""; + $body .= "The RICH gas system went in error state:\n"; + $body .= sprintf("Isobutan pressure : %.2f bar Error(<0.7, >2.0)\n",$IsobPress); + $body .= sprintf("O2 concentration : %.1f ppm Error(>5000.0)\n",$IsobO2); + $body .= sprintf("Isobutanoutput ratio : %.2f Error(<50.0)\n", $IsobRefR); + $body .= sprintf("Scales: %.1fkg / %.1fkg\n",,$IsobScale1,$IsobScale2); + Hmon::SendEmail($currEmailAddr,"GAS ERROR IN RICH",$body); + } + $mailoutErr = time(); + } + + my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time()); + if(!$QA::RichGasOff && $hour==12 && $min==15 && $mailoutDaily==0) { + foreach my $currEmailAddr (@emailAddrs) { + my $body = ""; + $body .= "It is 12:15 and time for your daily RICH gas mail:\n"; + $body .= sprintf("Isobutan pressure : %.2f bar\n",$IsobPress); + $body .= sprintf("O2 concentration : %.1f ppm\n",$IsobO2); + $body .= sprintf("Isobutanoutput ratio : %.2f\n", $IsobRefR); + $body .= sprintf("Scales: %.1fkg / %.1fkg\n",,$IsobScale1,$IsobScale2); + Hmon::SendEmail($currEmailAddr,"Daily RICH gas mail",$body); + } + $mailoutDaily = 1; + } + if(!$QA::RichGasOff && $hour==12 && $min==16) { + $mailoutDaily = 0; + } + if( $timer % 4 == 0) { $value = sprintf("%.1f|%.0f|%.0f",$IsobPress,$IsobO2,$IsobRefR); $longtext = sprintf("isobutan pressure : %.2f bar | O2 concentration: %.1f ppm
    |Isobutanoutput ratio: %.2f | Scales: %.1fkg / %.1fkg",$IsobPress,$IsobO2,$IsobRefR,$IsobScale1,$IsobScale2); diff --git a/hmon/permanent/hmon_rpchv.pl b/hmon/permanent/hmon_rpchv.pl index 0434fd3..a546fbd 100755 --- a/hmon/permanent/hmon_rpchv.pl +++ b/hmon/permanent/hmon_rpchv.pl @@ -67,7 +67,6 @@ while(1) { $cnt = ($i*2)+$j; if ($data->{"U$cnt"}->{val}>=5490){$ChnlsOn++;} elsif ($data->{"U$cnt"}->{val}>=4990){$ChnlsLow++;} - print $data->{"U$cnt"}->{val}."\n"; } } diff --git a/hmon/permanent/hmon_spillmon.pl b/hmon/permanent/hmon_spillmon.pl index fe698ec..ab78702 100755 --- a/hmon/permanent/hmon_spillmon.pl +++ b/hmon/permanent/hmon_spillmon.pl @@ -177,10 +177,13 @@ my $round = 0; #my @xpos = (15,13,11,9,7,5,3,1,0,2,4,6,8,10,12,14); #my @ypos = (0,2,4,6,8,10,12,14,15,13,11,9,7,5,3,1); -my @xpos = (1,3,5,7,9,11,13,15,14,12,10,8,6,4,2,0); +#my @xpos = (1,3,5,7,9,11,13,15,14,12,10,8,6,4,2,0); +my @xpos = (15,13,11,9,7,5,3,1,0,2,4,6,8,10,12,14); my @ypos = (15,13,11,9,7,5,3,1,0,2,4,6,8,10,12,14); + + while(1) { diff --git a/hmon/permanent/hmon_tofhv.pl b/hmon/permanent/hmon_tofhv.pl index 8655f8e..84f23c7 100755 --- a/hmon/permanent/hmon_tofhv.pl +++ b/hmon/permanent/hmon_tofhv.pl @@ -141,7 +141,7 @@ while(1) { $value = sprintf("HV is OFF"); $longtext = sprintf("HV is OFF"); } - QA::WriteQALog($flog,"hv","tofhv",30,$qastate,"TOF HV",$value,$longtext); + QA::WriteQALog($flog,"hv","tofhv",50,$qastate,"TOF HV",$value,$longtext); } diff --git a/hmon/qawindows/emc-main.htm b/hmon/qawindows/emc-main.htm index 405704a..6046203 100644 --- a/hmon/qawindows/emc-main.htm +++ b/hmon/qawindows/emc-main.htm @@ -12,7 +12,6 @@ , "HadesQA/EMC/hEmc_mult_raw_sec/hEmc_mult_raw_sec__0__0_+HadesQA/EMC/hEmc_mult_raw_sec/hEmc_mult_raw_sec__0__1_+HadesQA/EMC/hEmc_mult_raw_sec/hEmc_mult_raw_sec__0__2_+HadesQA/EMC/hEmc_mult_raw_sec/hEmc_mult_raw_sec__0__3_+HadesQA/EMC/hEmc_mult_raw_sec/hEmc_mult_raw_sec__0__4_+HadesQA/EMC/hEmc_mult_raw_sec/hEmc_mult_raw_sec__0__5_" , "HadesQA/EMC/hEmc_raw_sec" , "HadesQA/EMC/trending/hEmc_mult_raw_total_trend" - , "HadesQA/EMC/hRpcrawsecmult/hRpcrawsecmult__0__4_" , "HadesQA/EMC/trending/hEmc_mult_raw_sec_trend/hEmc_mult_raw_sec_trend__0__0_+HadesQA/EMC/trending/hEmc_mult_raw_sec_trend/hEmc_mult_raw_sec_trend__0__1_+HadesQA/EMC/trending/hEmc_mult_raw_sec_trend/hEmc_mult_raw_sec_trend__0__2_+HadesQA/EMC/trending/hEmc_mult_raw_sec_trend/hEmc_mult_raw_sec_trend__0__3_+HadesQA/EMC/trending/hEmc_mult_raw_sec_trend/hEmc_mult_raw_sec_trend__0__4_+HadesQA/EMC/trending/hEmc_mult_raw_sec_trend/hEmc_mult_raw_sec_trend__0__5_" ]' diff --git a/hmon/qawindows/mdcexpert-mb.htm b/hmon/qawindows/mdcexpert-mb.htm index 8997467..80d7481 100644 --- a/hmon/qawindows/mdcexpert-mb.htm +++ b/hmon/qawindows/mdcexpert-mb.htm @@ -1,25 +1,20 @@ -MDCExpert - T3 +MDCExpert - Mb - diff --git a/hmon/styles.css b/hmon/styles.css index 1edc9d1..ea3c244 100644 --- a/hmon/styles.css +++ b/hmon/styles.css @@ -25,6 +25,7 @@ body>div>div { } + div#content { background:transparent !important; /* transition: opacity .5s; */ @@ -466,7 +467,7 @@ svg { margin-top:10px; color:white; width:100%; - height:100px; + height:200px; font-size:13px; box-shadow:none !important; clear:both; @@ -628,3 +629,44 @@ border:0; padding:0; overflow:hidden; } + + + +#duck { + display:hidden; + } + +body:has(.QA) #duck { + position:absolute; + left:0; + bottom:0; + width:100%; + height:50px; + background-image:url('/mon/daqi.png'); + background-size:contain; + background-repeat:no-repeat; + background-position: 0 0; + will-change: transform; + /*animation: animatedBackground 60s infinite steps(100,end);*/ +} + +@keyframes animatedBackground { + 0% { + background-position: -10% 0; + transform: scaleX(1) translateZ(0); + } + 50% { + background-position: 110% 0; + transform: scaleX(1) translateZ(0); + } + 50.000001% { + background-position: -10% 0; + transform: scaleX(-1) translateZ(0); + } + 100% { + background-position: 110% 0; + transform: scaleX(-1) translateZ(0); + } +} + +