From a904cd1df290525175da6712f1becd841db2560f Mon Sep 17 00:00:00 2001 From: hadaq Date: Fri, 4 Feb 2022 21:38:02 +0100 Subject: [PATCH] update hmon files --- control/gui/expert/Thresholds_STS_Reset | 19 ++++++++ hmon/QA.pm | 11 +++-- hmon/doc/misc-seu.htt | 12 +++++ hmon/hmon_endpoints.pl | 13 +++--- hmon/hmon_logfrpc2.pl | 49 +++++++++++++++++++++ hmon/hmon_seu.pl | 58 +++++++++++++++++++++++++ hmon/hmon_tofrate.pl | 8 ++-- hmon/hmon_triggerratio.pl | 10 ++--- hmon/permanent/hmon_interrupt.pl | 0 hmon/permanent/hmon_logbook.pl | 3 +- hmon/permanent/hmon_mdchv.pl | 18 ++++---- hmon/permanent/hmon_mdcpressure.pl | 2 +- hmon/permanent/hmon_spillmon.pl | 2 +- hmon/start.sh | 3 +- 14 files changed, 176 insertions(+), 32 deletions(-) create mode 100755 control/gui/expert/Thresholds_STS_Reset create mode 100644 hmon/doc/misc-seu.htt create mode 100644 hmon/hmon_logfrpc2.pl create mode 100755 hmon/hmon_seu.pl mode change 100755 => 100644 hmon/permanent/hmon_interrupt.pl diff --git a/control/gui/expert/Thresholds_STS_Reset b/control/gui/expert/Thresholds_STS_Reset new file mode 100755 index 0000000..5dd8a42 --- /dev/null +++ b/control/gui/expert/Thresholds_STS_Reset @@ -0,0 +1,19 @@ +#!/bin/bash +export myvar=$(doyoureallywant.pl "to reset the STS thresholds?") +if [ $myvar == "256" ]; then exit; fi + +xterm -geometry 80x24 -bg orange -fg black -e bash --login -c ' + ssh hadaq@hadesp33 " + export DAQOPSERVER=hadesp31 + echo \" Connected to hades33...\" + echo \" Setting Thresholds...\" + cd /home/hadaq/trbsoft/hadesdaq/sts + + /home/hadaq/trbsoft/hadesdaq/sts/settings/reset_asics_sts1.sh + /home/hadaq/trbsoft/hadesdaq/sts/settings/reset_asics_sts2.sh + + echo \"Loading STS Pasttrec settings finished\" + sleep 10 +" +' + diff --git a/hmon/QA.pm b/hmon/QA.pm index f4757f7..425f89c 100644 --- a/hmon/QA.pm +++ b/hmon/QA.pm @@ -55,7 +55,7 @@ $entries->{'rich'} = ['temp','LV','bkpl','isob','richthr']; $entries->{'padiwa'} = ['ecal','start','itof','hodo','sts']; $entries->{'hv'} = ['richhv','ecalhv','rpchv','fwhv','tofhv']; $entries->{'hv2'} = ['mdchv','stshv','frpchv','starthv','itofhv']; -$entries->{'misc'} = ['irq','calib','magnet','env','sequencer']; +$entries->{'misc'} = ['seu','calib','magnet','env','sequencer']; $entries->{'pion'} = ['nxstatus', 'HV', 'HVcurr', 'cooling','seu']; @@ -102,8 +102,11 @@ our @EBDeltaRateLimits = (0.1, 0.2, 0.3); # as a ratio compared to the rate our $TrgCheckPolarity = 1; our @Eventsbroken = (.5,2,5); our @MdcTokenMissLimits = (10,50,100); -our @MdcNominalHV = (1750,1770,1900,2150); -our $MdcHVOffsetLimits = [[5,10,255],[5,10,255],[5,10,255],[10,110,325]]; +##single HV each plane +#our @MdcNominalHV = (1750,1770,1900,2150); +#individual HV each chamber +our $MdcNominalHV = [[1750,1750,1750,1750,1750,1750],[1770,1790,1770,1770,1770,1770],[1900,1900,1900,1900,1900,1900],[2150,2150,2150,2150,2150,2150]]; +our $MdcHVOffsetLimits = [[5,10,255],[5,30,255],[5,10,255],[10,110,325]]; our @EcalHvLimits = (493,325,320); our @RpcHvLimits = (12,12,12); our @TofHvLimits = (764,760,759); @@ -155,7 +158,7 @@ our $QAServerOff = 0; # Missing Boards ############################################################################### -our @mdc_boards_removed =(0x2203); #(0x2257, 0x2029); # 2233 added 2014-08-28 , 2203 at 2014/09/03, 2029 2014/09/23 +our @mdc_boards_removed =(0x2203, 0x2027, 0x2057); #(0x2257, 0x2029); # 2233 added 2014-08-28 , 2203 at 2014/09/03, 2029 2014/09/23 our @mdc_chambers_removed =(); our @rich_boards_removed =(); our @tof_boards_removed =(); diff --git a/hmon/doc/misc-seu.htt b/hmon/doc/misc-seu.htt new file mode 100644 index 0000000..7edc8a3 --- /dev/null +++ b/hmon/doc/misc-seu.htt @@ -0,0 +1,12 @@ +

Single Event Upsets in FPGA

+Radition effects can change the configuration of FPGA and alter their behavior. +Few of them are able to monitor these changes. The number shown is a rough estimate +for the severity of radiation effects in the system. Note that neither MDC nor RICH +front-ends (the most exposed systems!) can be included here. + + +

Error Handling

+ +The full message contains the addresses of affected boards - consider rebooting the +coresponding subsystem at your convenience. Doing so before an actual failure of +the board is always simpler and quicker. diff --git a/hmon/hmon_endpoints.pl b/hmon/hmon_endpoints.pl index e274c8c..2f07962 100755 --- a/hmon/hmon_endpoints.pl +++ b/hmon/hmon_endpoints.pl @@ -9,7 +9,7 @@ use QA; use Getopt::Long; use Data::Dumper; use List::MoreUtils qw(uniq); -use List::Util qw(max); +use List::Util qw(min max); use HADES::TrbNet; trb_init_ports() or die trb_strerror(); @@ -254,7 +254,7 @@ while(1) { $mdcmissingtime += max(0,$num_mdc_missing-2); #Are two boards from the same chamber missing? - my @mdcboards = @sorted_mdc_results; + my @mdcboards = (@sorted_mdc_results,@QA::mdc_boards_removed); chop(@mdcboards); my @uniqchambers = uniq @mdcboards; my $samechamberboards = 0; @@ -272,15 +272,16 @@ while(1) { my $title = "MDC"; my $value = "OK ".(scalar @mdc_boards - scalar @QA::mdc_boards_removed); + $value .= "(-".(scalar @QA::mdc_boards_removed).")" if scalar @QA::mdc_boards_removed; if ($num_mdc_missing > 0) {$value = "$num_mdc_missing / ".(scalar @mdc_boards - scalar @QA::mdc_boards_removed)." miss";} if ($num_mdc_mistake > 0) {$value = "Check Script";} - my $longtext = (scalar @mdc_boards). " boards. ".(scalar @QA::mdc_boards_removed)." removed"; - if ($num_mdc_missing > 0) {$longtext = "Endp @sorted_mdc_results missing"}; + my $longtext = (scalar @mdc_boards). " boards. ".(scalar @QA::mdc_boards_removed)." removed
"; + if ($num_mdc_missing > 0) {$longtext .= "Endp @sorted_mdc_results missing
"}; if ($num_mdc_mistake > 0) {$longtext .= " Endp @{$mdc_results[4]} not known";} my $qastate = QA::GetQAState('below',$num_mdc_missing,@QA::MdcEndpMissingLimits); - $qastate = QA::ERROR if $samechamberboards; - $qastate = QA::WARN if $samesectorboards; + $qastate = max($qastate,QA::ERROR) if $samechamberboards; + $qastate = min(QA::WARN,$qastate) if !$samesectorboards; $qastate = QA::ERROR if $mdcmissingtime > 360; $longtext .= "
Few boards missing, but for a long time. Better do a restart." if $mdcmissingtime > 360; diff --git a/hmon/hmon_logfrpc2.pl b/hmon/hmon_logfrpc2.pl new file mode 100644 index 0000000..3cf80fb --- /dev/null +++ b/hmon/hmon_logfrpc2.pl @@ -0,0 +1,49 @@ +#!/usr/bin/perl + +use warnings; +use strict; +use Data::Dumper; + +use HADES::TrbNet; +use POSIX qw(strftime); +my $fh; + + +trb_init_ports() or die trb_strerror(); +my $inspill; +my $old; +while (1) { + + my $o = trb_register_read(0x10,0x905d); + $inspill = $o->{0x10} >> 31; + + $o = trb_register_read_mem(0xfe47,0xdfc0,0,32); + if(defined $old) { + my @diff; + + foreach my $b (sort keys %$o) { + next if ($b < 0x6800 || $b > 0x6813); + for my $v (0..31) { + my $d = ($o->{$b}[$v]&0xffffff)-($old->{$b}[$v]&0xffffff); + $d += 2**24 if $d < 0; + push(@diff, $d); + } + } + + my $day = strftime("%Y%m%d",localtime()); + my $str = ""; + $str .= strftime("%Y.%m.%d %H:%M:%S",localtime())."\t"; + $str .= time()."\t"; + $str .= $inspill."\t"; + for my $i (0..scalar @diff -1) { + $str .= sprintf("%u\t",$diff[$i]); + } + $str .= "\n"; + open($fh, ">>","/home/hadaq/trbsoft/daq/hmon/archive/frpcscalerloginbeam-$day.csv"); + print $fh $str; + close $fh; + } + + $old = $o; + sleep(1); + } diff --git a/hmon/hmon_seu.pl b/hmon/hmon_seu.pl new file mode 100755 index 0000000..884cc16 --- /dev/null +++ b/hmon/hmon_seu.pl @@ -0,0 +1,58 @@ +#!/usr/bin/perl + +use warnings; +use strict; +use Data::Dumper; +use Data::TreeDumper; +use Hmon; +use QA; +use HADES::TrbNet; +use HPlot; +use Time::HiRes qw(usleep time); +use List::Util qw[min max]; + +use Encode qw(encode from_to); +use JSON::XS; + +my $fqa = QA::OpenQAFile(); +trb_init_ports() or die trb_strerror(); + + + + +trb_register_write(0xfe61,0xd500,1); +trb_register_write(0xfe65,0xd500,1); +trb_register_write(0xfe66,0xd500,1); +trb_register_write(0xfe73,0xd500,1); +trb_register_write(0xfe74,0xd500,1); +trb_register_write(0xfe76,0xd500,1); +trb_register_write(0xfe71,0xd500,1); +trb_register_write(0xfe40,0xd500,1); +trb_register_write(0xfe4C,0xd500,1); +trb_register_write(0xfe47,0xd500,1); +trb_register_write(0xfe52,0xd500,1); + +while(1) { + my $totalboards = 0; + my $seuboards = 0; + my @boards; + my $longtext = ""; + my $value = ""; + my $qastate = QA::OK; + + my $o = trb_register_read(0xffff,0xd501); + foreach my $b (keys %$o) { + next if $o->{$b} == 0x00000008; + next if $o->{$b} == 0x00000000; + $totalboards++; + next unless $o->{$b} & 0xFF000000; + $seuboards++; + push(@boards,$b); + } + $longtext = "$seuboards out of $totalboards monitored report some radiation issues
"; + $longtext .= sprintf("%04x ",$_) for sort @boards; + $value = "$seuboards/$totalboards"; + QA::WriteQALog($fqa,"misc","seu",30,$qastate,'SEU',$value,$longtext); + print $value." ".$longtext."\n"; + sleep 10; + } diff --git a/hmon/hmon_tofrate.pl b/hmon/hmon_tofrate.pl index 1ce074a..1e2ffe9 100755 --- a/hmon/hmon_tofrate.pl +++ b/hmon/hmon_tofrate.pl @@ -56,9 +56,9 @@ my $diff; my $iter = 0; my $lowratelimit = 200; -my $highratelimit = 50000; -my $lowratecount = 500; -my $highratecount = 30; +my $highratelimit = 30000; +my $lowratecount = 400; +my $highratecount = 10; my $qastate = QA::OK; my $text = "OK"; @@ -95,7 +95,7 @@ while(1) { HPlot::PlotDraw('TofRate'); } - +# print $highrate." ".$lowrate."\n"; if($lowrate > $lowratecount) { $qastate = QA::OK; diff --git a/hmon/hmon_triggerratio.pl b/hmon/hmon_triggerratio.pl index 134869f..b6f76bd 100755 --- a/hmon/hmon_triggerratio.pl +++ b/hmon/hmon_triggerratio.pl @@ -160,8 +160,8 @@ while (1) { # print $startdirect." ".$start."\n"; ###Ratio M2 over Start - my $ratio = $m2/($start||1); - if ($start <= $STARTRATELIMIT || !$inspill) { + my $ratio = $m2/($startdirect||1); + if ($startdirect <= $STARTRATELIMIT || !$inspill) { HPlot::PlotAdd("M2StartRatio",0,1); HPlot::PlotAdd("M2StartRatio","NaN",0) } @@ -176,8 +176,8 @@ while (1) { ###Ratio PT3 over Start - $ratio = $pt3/($start||1); - if ($start <= $STARTRATELIMIT || !$inspill) { + $ratio = $pt3/($startdirect||1); + if ($startdirect <= $STARTRATELIMIT || !$inspill) { HPlot::PlotAdd("Pt3StartRatio",0,1); HPlot::PlotAdd("Pt3StartRatio","NaN",0) } @@ -238,7 +238,7 @@ while (1) { $lastinspill = $inspill; $iter++; $iter = 0 if $iter == 6; - print time()."\n"; +# print time()."\n"; usleep(200000); } diff --git a/hmon/permanent/hmon_interrupt.pl b/hmon/permanent/hmon_interrupt.pl old mode 100755 new mode 100644 diff --git a/hmon/permanent/hmon_logbook.pl b/hmon/permanent/hmon_logbook.pl index 8d0c9a9..99157af 100755 --- a/hmon/permanent/hmon_logbook.pl +++ b/hmon/permanent/hmon_logbook.pl @@ -12,13 +12,14 @@ use JSON; my $firstrun =1; -my $command = "curl 'https://hades-db.gsi.de/pls/hades_webdbs/hades_oper.hlogbook2.show' -X POST -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' -H 'Accept-Language: de,en-US;q=0.7,en;q=0.3' -H 'Content-Type: application/x-www-form-urlencoded' -H 'Origin: https://hades-db.gsi.de' -H 'DNT: 1' -H 'Connection: keep-alive' -H 'Referer: https://hades-db.gsi.de/pls/hades_webdbs/hades_oper.hlogbook2.form_selection' -H 'Upgrade-Insecure-Requests: 1' -H 'Sec-Fetch-Dest: iframe' -H 'Sec-Fetch-Mode: navigate' -H 'Sec-Fetch-Site: same-origin' -H 'Sec-Fetch-User: ?1' --data-raw 'p_exp_id=FEB22&p_begin=&p_end=&p_num_days=&p_num_entries=8&p_author=&p_action=Show&p_log_type=All&p_run_type=None&p_current=&p_filename=*&p_search_text=*' -u hades:6\\\$ectors 2>/dev/null"; +my $command = "curl 'https://hades-db.gsi.de/pls/hades_webdbs/hades_oper.hlogbook2.show' -b /tmp/cookiefile -c /tmp/cookiefile -X POST -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' -H 'Accept-Language: de,en-US;q=0.7,en;q=0.3' -H 'Content-Type: application/x-www-form-urlencoded' -H 'Origin: https://hades-db.gsi.de' -H 'DNT: 1' -H 'Connection: keep-alive' -H 'Referer: https://hades-db.gsi.de/pls/hades_webdbs/hades_oper.hlogbook2.form_selection' -H 'Upgrade-Insecure-Requests: 1' -H 'Sec-Fetch-Dest: iframe' -H 'Sec-Fetch-Mode: navigate' -H 'Sec-Fetch-Site: same-origin' -H 'Sec-Fetch-User: ?1' --data-raw 'p_exp_id=FEB22&p_begin=&p_end=&p_num_days=&p_num_entries=8&p_author=&p_action=Show&p_log_type=All&p_run_type=None&p_current=&p_filename=*&p_search_text=*' -u hades:6\\\$ectors "; my $store; my $hash; $hash->{username}='daqi'; while(1) { + print time()."\n"; my @res = qx($command); #webserver uses iso-8859-1 -( # print Dumper @res; diff --git a/hmon/permanent/hmon_mdchv.pl b/hmon/permanent/hmon_mdchv.pl index 932e006..9b8c207 100755 --- a/hmon/permanent/hmon_mdchv.pl +++ b/hmon/permanent/hmon_mdchv.pl @@ -198,7 +198,7 @@ while(1) { $mdcHvMax[$p] = $volt; } - if($current > 90.0 || (!($p == 2 && $s == 4) && $current > 80.0) ) { + if($current > 20.0 ) { $spikeString .= "P$p S$s F/C $i: HV = ".$volt." , I = ".$current." - ".localtime()."\n"; $currenterr++; } @@ -207,9 +207,9 @@ while(1) { my $ch_qastate = QA::OK; my $chamberHVmin = min($data->{"$p-$s-0-V"}->{val},$data->{"$p-$s-1-V"}->{val}); - if($chamberHVmin < $QA::MdcNominalHV[$p-1] - $QA::MdcHVOffsetLimits->[$p-1][0]) {$qastate = QA::WARN;} - if($chamberHVmin < $QA::MdcNominalHV[$p-1] - $QA::MdcHVOffsetLimits->[$p-1][1]) {$qastate = QA::WARN_2;} - if($chamberHVmin < $QA::MdcNominalHV[$p-1] - $QA::MdcHVOffsetLimits->[$p-1][2]) {$qastate = QA::ERROR;} + if($chamberHVmin < $QA::MdcNominalHV->[$p-1][$s-1] - $QA::MdcHVOffsetLimits->[$p-1][0]) {$ch_qastate = QA::WARN;} + if($chamberHVmin < $QA::MdcNominalHV->[$p-1][$s-1] - $QA::MdcHVOffsetLimits->[$p-1][1]) {$ch_qastate = QA::WARN_2;} + if($chamberHVmin < $QA::MdcNominalHV->[$p-1][$s-1] - $QA::MdcHVOffsetLimits->[$p-1][2]) {$ch_qastate = QA::ERROR;} if ($ch_qastate >= QA::WARN_2) { $errortimer[$chamnum]++; @@ -222,11 +222,11 @@ while(1) { } push(@qastate_chamber,$ch_qastate); - } - if($mdcHvMax[$p] > $QA::MdcNominalHV[$p-1] + $QA::MdcHVOffsetLimits->[$p-1][0]) {$qastate = max($qastate,QA::WARN);} - if($mdcHvMax[$p] > $QA::MdcNominalHV[$p-1] + $QA::MdcHVOffsetLimits->[$p-1][1]) {$qastate = max($qastate,QA::WARN_2);} - if($mdcHvMax[$p] > $QA::MdcNominalHV[$p-1] + $QA::MdcHVOffsetLimits->[$p-1][2]) {$qastate = max($qastate,QA::ERROR);} + # if($mdcHvMax[$p] > $QA::MdcNominalHV[$p-1][$s-1] + $QA::MdcHVOffsetLimits->[$p-1][0]) {$qastate = max($qastate,QA::WARN);} + # if($mdcHvMax[$p] > $QA::MdcNominalHV[$p-1][$s-1] + $QA::MdcHVOffsetLimits->[$p-1][1]) {$qastate = max($qastate,QA::WARN_2);} + # if($mdcHvMax[$p] > $QA::MdcNominalHV[$p-1][$s-1] + $QA::MdcHVOffsetLimits->[$p-1][2]) {$qastate = max($qastate,QA::ERROR);} + } } $qastate = max($qastate,@qastate_chamber); @@ -242,7 +242,7 @@ sprintf("%.1f/%.1f/%.1f/%.1f",$mdcHvMin[1]/1000.,$mdcHvMin[2]/1000.,$mdcHvMin[3] "
Nominal [V]: %i / %i / %i / %i", $mdcHvMin[1],$mdcHvMin[2],$mdcHvMin[3],$mdcHvMin[4], $mdcHvMax[1],$mdcHvMax[2],$mdcHvMax[3],$mdcHvMax[4], - $QA::MdcNominalHV[0],$QA::MdcNominalHV[1],$QA::MdcNominalHV[2],$QA::MdcNominalHV[3] + min(@{$QA::MdcNominalHV->[0]}),min(@{$QA::MdcNominalHV->[1]}),min(@{$QA::MdcNominalHV->[2]}),min(@{$QA::MdcNominalHV->[3]}) ); QA::WriteQALog($flog,"hv2","mdchv",10,$qastate,"MDC HV",$value,$longtext); diff --git a/hmon/permanent/hmon_mdcpressure.pl b/hmon/permanent/hmon_mdcpressure.pl index 111364f..73b2c98 100755 --- a/hmon/permanent/hmon_mdcpressure.pl +++ b/hmon/permanent/hmon_mdcpressure.pl @@ -262,7 +262,7 @@ Perl2Epics::Connect("p1CO2","HAD:MDC:GAS:CO2:concentration:average"); Perl2Epics::Connect("p2CO2","HAD:MDC:GAS:CO2:concentration2:average"); Perl2Epics::Connect("Vachigh","HAD:VAC:tpg366-1:ch4"); -Perl2Epics::Connect("Vaclow" ,"HAD:VAC:tpg366-1:ch1"); +Perl2Epics::Connect("Vaclow" ,"HAD:VAC:tpg366-1:ch2"); Perl2Epics::Connect("Gate" ,"HAD:VAC:beamlineGate"); print "Connected.\n"; diff --git a/hmon/permanent/hmon_spillmon.pl b/hmon/permanent/hmon_spillmon.pl index 1d72358..1cd1c18 100755 --- a/hmon/permanent/hmon_spillmon.pl +++ b/hmon/permanent/hmon_spillmon.pl @@ -100,7 +100,7 @@ Hmon::WriteFile("MicroSpillStructure",$str); trb_init_ports() or die trb_strerror(); -trb_register_write(0xffff,0xb805,30000); #50000 = 1s offset to spill start +trb_register_write(0xffff,0xb805,200000); #50000 = 1s offset to spill start my $round = 0; while(1) { diff --git a/hmon/start.sh b/hmon/start.sh index e5621a7..4cd4645 100755 --- a/hmon/start.sh +++ b/hmon/start.sh @@ -45,7 +45,8 @@ mkdir -p `pwd`/pion pkill -f "sshfs -o allow_other hadaq@hadesp66:/home/hades-qa/online/5.34.38/pics" sshfs -o allow_other hadaq@hadesp66:/home/hades-qa/online/5.34.38/pics `pwd`/qa -sshfs -o allow_other hadaq@hades66:/home/hades-qa/online/5.34.38/feb22/vertex/pics `pwd`/vertex +fusermount -u vertex +sshfs -o allow_other hadaq@hadesp66:/home/hades-qa/online/5.34.38/feb22/vertex/pics `pwd`/vertex # sshfs -o allow_other hadaq@lxhadeb06p:/data01/tmp/ `pwd`/pion pkill -f "sshfs -o allow_other hadaq@hadesp63:/home/hadaq/local/tdcmon/dabc/cal" -- 2.43.0