--- /dev/null
+#!/usr/bin/perl -w
+
+use English;
+use strict;
+use Getopt::Long;
+use Data::Dumper;
+use Config::Std;
+use FileHandle;
+use List::MoreUtils qw(any apply first_index);
+use File::Basename;
+use Cwd;
+
+#- Copy all the arguments because
+# later on the @ARGV becomes empty
+my @arg_list = @ARGV;
+
+#- the command line option flags
+my $opt_help = 0;
+my $opt_ebconf = "/home/hadaq/trbsoft/hadesdaq/evtbuild/eb.conf";
+my $opt_ioc = "";
+my $opt_test = 0;
+my $opt_verb = 0;
+my $opt_eb = "";
+my @opt_ebrange = ();
+my $opt_rfio = 'undef';
+my $opt_disk = 'undef';
+my $opt_online = 'undef';
+my $opt_bnet = 3;
+my $opt_prefix;
+
+GetOptions ('h|help' => \$opt_help,
+ 'c|conf=s' => \$opt_ebconf,
+ 'e|eb=s' => \$opt_eb,
+ 'i|ioc=s' => \$opt_ioc,
+ 't|test' => \$opt_test,
+ 'n|nr=s' => \@opt_ebrange,
+ 'd|disk=s' => \$opt_disk,
+ 'r|rfio=s' => \$opt_rfio,
+ 'p|prefix=s' => \$opt_prefix,
+ 'o|online=s' => \$opt_online,
+ 'b|bnet=s' => \$opt_bnet,
+ 'v|verb' => \$opt_verb);
+
+if( $opt_help ) {
+ &help();
+ exit(0);
+}
+
+#- List of EBs provided via command line options
+my $active_EBs_aref = &setArgs();
+
+#- Hash with status of CPU cores of EBs (used for 'taskset')
+my %EB_CPU_status;
+my $EB_CPU_status_href = \%EB_CPU_status;
+&init_CPU_status($EB_CPU_status_href);
+
+my $expect_ioc_script = "/tmp/ioc_exit.exp";
+my $log_path = "/tmp/log";
+my %temp_args;
+my $temp_args_href = \%temp_args;
+read_config $opt_ebconf => %$temp_args_href;
+#rint Dumper $temp_args_href;
+#exit;
+
+
+my @bnetservers = ("192.168.100.8","192.168.100.9","192.168.100.10","192.168.105.11", "192.168.105.12");
+
+#my @bnetservers = ("192.168.100.10","192.168.100.9","192.168.100.8","192.168.105.11", "192.168.105.12");
+
+
+
+# my %IP_GSI_lookup = (
+# '192.168.100.15' => '140.181.66.160',
+# '192.168.100.12' => '140.181.76.94',
+# '192.168.100.13' => '140.181.91.154',
+# '192.168.100.14' => '140.181.88.20',
+# '192.168.100.8' => '140.181.80.62',
+# '192.168.100.9' => '140.181.80.64',
+# '192.168.100.10' => '140.181.80.68',
+# '192.168.105.11' => '140.181.83.135',
+# '192.168.105.12' => '140.181.83.168',
+# '192.168.105.13' => '140.181.83.176'
+# );
+
+
+ my %IP_GSI_lookup = (
+ '192.168.100.15' => 'lxhadeb05',
+ '192.168.100.12' => 'lxhadeb02',
+ '192.168.100.13' => 'lxhadeb03',
+ '192.168.100.14' => 'lxhadeb04',
+ '192.168.100.8' => 'lxhadeb08',
+ '192.168.100.9' => 'lxhadeb09',
+ '192.168.100.10' => 'lxhadeb10',
+ '192.168.105.11' => 'lxhadeb11',
+ '192.168.105.12' => 'lxhadeb12',
+ '192.168.105.13' => 'lxhadeb13'
+ );
+
+
+my $IP_GSI_href = \%IP_GSI_lookup;
+
+
+my $numOfEBProcs = 0;
+my %EB_Args;
+my $EB_Args_href = \%EB_Args;
+
+#my @EB_IP_list;
+
+
+
+&getEBArgs( $EB_Args_href );
+
+if($opt_ioc eq "start"){
+ &killIOC();
+ &startIOC();
+}
+elsif($opt_ioc eq "stop"){
+ &killIOC();
+}
+elsif($opt_eb eq "start"){
+ &writeArgs2file();
+ &startEvtBuilders();
+}
+elsif($opt_eb eq "stop"){
+ &stopEvtBuilders();
+}
+elsif($opt_eb eq "restart"){
+ &stopEvtBuilders();
+ sleep 1;
+ &writeArgs2file();
+ &startEvtBuilders();
+}
+
+exit(0);
+
+################### END OF MAIN ####################
+
+sub help()
+{
+ print "\n";
+ print << 'EOF';
+start_eb_gbe.pl
+
+ This script starts parallel Event Building processes.
+ The script also starts IOC processes for the run control.
+
+Usage:
+
+ Command line: start_eb_gbe.pl
+ [-h|--help] : Show this help.
+ [-c|--conf <path/name>] : Path to the config file (default: ../evtbuild/eb.conf).
+ [-e|--eb <start|stop|restart>] : Start or stop Event Builders (default: start).
+ [-i|--ioc <start|stop>] : Start or stop IOCs (default: start).
+ [-n|--nr <rangeOfEBs>] : Range of numbers of Event Bulders to be started.
+ [-d|--disk <on|off>] : Switch writing to disk on|off.
+ [-r|--rfio <on|off>] : Switch writing to tape on|off.
+ [-p|--prefix <prefix>] : Prefix of hld file.
+ [-o|--online <on|off>] : Switch RPC server on|off.
+ [-b|--bnet <on|off>] : Specify number of bnet builder nodes(default: 3).
+ [-t|--test] : Test without execution.
+ [-v|--verb] : More verbouse.
+
+Examples:
+
+ Start 6 EBs with the numbers 1,2,3,5,7 and prefix 'md':
+ start_eb_gbe.pl -e start -n 1-3 -n 5 -n 7 -p md
+
+ Start EBs and enable writing to disks but disable writing to tape for all EBs:
+ start_eb_gbe.pl -e start --disk on --rfio off
+
+EOF
+}
+
+sub init_CPU_status()
+{
+
+
+# JAM 17-oct-2018 - for BNET this cpu reservation is deprecated TODO: remove it
+ my ($EB_CPU_status_href) = @_;
+
+ # CPU affinity with 'taskset'
+ #
+ # CPU dec bin hex
+ # 0 1 1
+ # 1 10 2
+ # 2 100 4
+ # 3 1000 8
+ # 4 10000 10
+
+ #cores 0/1 reserved for system 02-05
+ #cores 2/3 reserved for interrupts on 02-05
+
+#lxhadeb01 is gone
+# foreach my $core (0..7){
+# if($core == 1){
+# $EB_CPU_status_href->{'192.168.100.11'}->{$core} = "res"; #reserved
+# }
+# else{
+# $EB_CPU_status_href->{'192.168.100.11'}->{$core} = "free";
+# }
+# }
+
+#JAM adjust this to actual affinities for eth0 settings TODO
+# eth0 interrupts are above core 8 now
+#
+#
+# foreach my $core (0..11){
+# if(($core < 2) || ($core> 8 ) ){
+# $EB_CPU_status_href->{'192.168.100.12'}->{$core} = "res"; #reserved
+# $EB_CPU_status_href->{'192.168.100.13'}->{$core} = "res"; #reserved
+# $EB_CPU_status_href->{'192.168.100.14'}->{$core} = "res"; #reserved
+# }
+# else{
+# $EB_CPU_status_href->{'192.168.100.12'}->{$core} = "free";
+# $EB_CPU_status_href->{'192.168.100.13'}->{$core} = "free";
+# $EB_CPU_status_href->{'192.168.100.14'}->{$core} = "free";
+# }
+# }
+#
+## after upgrade to debian 7: ethernet reserved cores are below 6
+ foreach my $core (0..11){
+ if(($core < 6) ){
+ $EB_CPU_status_href->{'192.168.100.12'}->{$core} = "res"; #reserved
+ $EB_CPU_status_href->{'192.168.100.13'}->{$core} = "res"; #reserved
+ $EB_CPU_status_href->{'192.168.100.14'}->{$core} = "res"; #reserved
+ }
+ else{
+ $EB_CPU_status_href->{'192.168.100.12'}->{$core} = "free";
+ $EB_CPU_status_href->{'192.168.100.13'}->{$core} = "free";
+ $EB_CPU_status_href->{'192.168.100.14'}->{$core} = "free";
+
+ }
+ }
+
+# eth0 ir is set above core 11
+ foreach my $core (0..23){
+ if( ($core < 2) ||( $core > 11) ){
+ $EB_CPU_status_href->{'192.168.100.15'}->{$core} = "res"; #reserved
+ }
+ else{
+ $EB_CPU_status_href->{'192.168.100.15'}->{$core} = "free";
+ }
+ }
+
+ # add cores for new EB servers JAM2018
+ foreach my $core (0..63){
+ if( ($core < 2) ||( $core > 60) ){
+ $EB_CPU_status_href->{'192.168.100.8'}->{$core} = "res"; #reserved
+ $EB_CPU_status_href->{'192.168.100.9'}->{$core} = "res"; #reserved
+ $EB_CPU_status_href->{'192.168.100.10'}->{$core} = "res"; #reserved
+ $EB_CPU_status_href->{'192.168.105.11'}->{$core} = "res"; #reserved
+ $EB_CPU_status_href->{'192.168.105.12'}->{$core} = "res"; #reserved
+ }
+ else{
+ $EB_CPU_status_href->{'192.168.100.8'}->{$core} = "free";
+ $EB_CPU_status_href->{'192.168.100.9'}->{$core} = "free";
+ $EB_CPU_status_href->{'192.168.100.10'}->{$core} = "free";
+ $EB_CPU_status_href->{'192.168.105.11'}->{$core} = "free";
+ $EB_CPU_status_href->{'192.168.105.12'}->{$core} = "free";
+
+ }
+ }
+ # alternative setup: specifiy cpu bitmasks for different roles:
+
+ $EB_CPU_status_href->{'192.168.100.15'}->{'input'} = "2-23";
+ $EB_CPU_status_href->{'192.168.100.15'}->{'build'} = "2-23";
+
+ # JAM12-2018: reserve lower 32 cpus for network interrupts
+ $EB_CPU_status_href->{'192.168.100.8'}->{'input'} = "32-47";
+ $EB_CPU_status_href->{'192.168.100.8'}->{'build'} = "48-63";
+ $EB_CPU_status_href->{'192.168.100.9'}->{'input'} = "32-47";
+ $EB_CPU_status_href->{'192.168.100.9'}->{'build'} = "48-63";
+ $EB_CPU_status_href->{'192.168.100.10'}->{'input'} = "32-47";
+ $EB_CPU_status_href->{'192.168.100.10'}->{'build'} = "48-63";
+ $EB_CPU_status_href->{'192.168.105.11'}->{'input'} = "32-47";
+ $EB_CPU_status_href->{'192.168.105.11'}->{'build'} = "48-63";
+ $EB_CPU_status_href->{'192.168.105.12'}->{'input'} = "32-47";
+ $EB_CPU_status_href->{'192.168.105.12'}->{'build'} = "48-63";
+
+}
+
+
+sub getCoreNr()
+{
+ #my ($ip) = @_;
+ my ($ip, $role) = @_;
+ my $core_nr;
+ if(defined $role)
+ {
+
+ $core_nr = $EB_CPU_status_href->{$ip}->{$role}
+
+ }
+ else
+ {
+ ################### begin old
+
+ foreach my $eb_ip (sort keys %$EB_CPU_status_href){
+ next unless($ip eq $eb_ip);
+
+ foreach my $core ( sort {$a <=> $b} keys %{$EB_CPU_status_href->{$eb_ip}} ){
+ my $core_status = $EB_CPU_status_href->{$eb_ip}->{$core};
+
+ next unless(lc($core_status) eq "free");
+
+ $core_nr = $core;
+ $EB_CPU_status_href->{$eb_ip}->{$core} = "busy";
+ last;
+ }
+ }
+
+ #- If no free cores left - take reserved cores
+ unless( defined $core_nr ){
+ foreach my $eb_ip (sort keys %$EB_CPU_status_href){
+ next unless($ip eq $eb_ip);
+
+ foreach my $core ( sort {$a <=> $b} keys %{$EB_CPU_status_href->{$eb_ip}} ){
+ my $core_status = $EB_CPU_status_href->{$eb_ip}->{$core};
+
+ if(lc($core_status) eq "res"){
+ $core_nr = $core;
+ $EB_CPU_status_href->{$eb_ip}->{$core} = "busy";
+ last;
+ }
+ }
+ }
+ }
+
+################### end old
+}
+ unless( defined $core_nr ){
+ print "No free cores left on CPU $ip. Exit.\n";
+ exit(0);
+ }
+
+ print "found core numbers $core_nr for node $ip, role $role\n";
+ return $core_nr;
+}
+
+
+
+
+sub setArgs()
+{
+ my @active_EBs;
+
+ if(@opt_ebrange){
+ foreach my $range (@opt_ebrange){
+ if($range =~ /(\d+)-(\d+)/){
+ my $max = $1;
+ my $min = $2;
+
+ foreach my $eb ($max..$min){
+ #- 1 must be subtracted to match
+ # EB numbering in the register_configgbe_ip.db
+ # which starts from zero
+ &checkEB_nr($eb);
+ push(@active_EBs, $eb-1);
+ }
+ }
+ elsif($range =~ /(\d+)/){
+ &checkEB_nr($1);
+ push(@active_EBs, $1-1);
+ }
+ }
+ }
+
+ return \@active_EBs;
+}
+
+sub checkEB_nr()
+{
+ my ($eb_nr) = @_;
+
+ if( $eb_nr < 1 || $eb_nr > 16 ){
+ print "ERROR: EB number should be in the range 1-16. Exit.";
+ exit(0);
+ }
+}
+
+sub getEBArgs()
+{
+ my ($href) = @_;
+
+ my $prefix = $temp_args_href->{'Main'}->{'EB_EXT'};
+ $prefix = $opt_prefix if( defined $opt_prefix );
+ my $filesize = $temp_args_href->{'Main'}->{'EB_FSIZE'};
+
+ my $base_port = $temp_args_href->{'Parallel'}->{'BASE_PORT'};
+ my $shift_port = $temp_args_href->{'Parallel'}->{'SHIFT_PORT'};
+ my $source_num = $temp_args_href->{'Parallel'}->{'NUM_OF_SOURCES'};
+ my $queuesize = $temp_args_href->{'Parallel'}->{'QUEUESIZE'};
+
+ my $multidisk = $temp_args_href->{'Parallel'}->{'MULTIDISK'};
+
+ #- Number of EB process
+ my $ebproc = 0;
+
+ #- List of BEs
+ my $listOfEBs = $temp_args_href->{'Parallel'}->{'EB_LIST'};
+ my @eb_list = split(/\s+/, $listOfEBs);
+
+ #- DABC mode selection
+ my $listOfDABC = $temp_args_href->{'Parallel'}->{'DABC'};
+ my @dabc_list = split(/\s+/, $listOfDABC);
+
+
+ # BNET setup:
+ my $listOfBnetInputs = $temp_args_href->{'Parallel'}->{'BNETINP'};
+ my @bnet_in_list = split(/\s+/, $listOfBnetInputs);
+
+ my $listOfBnetBuilders = $temp_args_href->{'Parallel'}->{'BNETBLD'};
+ my @bnet_bld_list = split(/\s+/, $listOfBnetBuilders);
+ #print Dumper @bnet_bld_list;
+
+
+ #- Default RFIO settings
+ my $rfio = $temp_args_href->{'Parallel'}->{'RFIO'};
+ my $rfio_path = $temp_args_href->{'Parallel'}->{'RFIO_PATH'};
+ my $rfio_pcOptions = $temp_args_href->{'Parallel'}->{'RFIO_pcOptions'};
+ my $rfio_iCopyMode = $temp_args_href->{'Parallel'}->{'RFIO_iCopyMode'};
+ my $rfio_pcCopyPath = $temp_args_href->{'Parallel'}->{'RFIO_pcCopyPath'};
+ my $rfio_iCopyFrac = $temp_args_href->{'Parallel'}->{'RFIO_iCopyFraction'};
+ my $rfio_iMaxFile = $temp_args_href->{'Parallel'}->{'RFIO_iMaxFile'};
+ my $rfio_iPathConv = $temp_args_href->{'Parallel'}->{'RFIO_iPathConvention'};
+
+ my @rfio_list = split(/\s+/, $rfio);
+
+
+
+ #- LTSM settings
+ my $listOfLTSM = $temp_args_href->{'Parallel'}->{'LTSM'};
+ my @ltsm_list = split(/\s+/, $listOfLTSM);
+
+ my $ltsm_path = $temp_args_href->{'Parallel'}->{'LTSM_PATH'};
+ my $ltsm_server = $temp_args_href->{'Parallel'}->{'LTSM_Server'};
+ my $ltsm_node = $temp_args_href->{'Parallel'}->{'LTSM_Node'};
+ my $ltsm_passwd = $temp_args_href->{'Parallel'}->{'LTSM_Passwd'};
+ my $ltsm_filesys = $temp_args_href->{'Parallel'}->{'LTSM_Filesystem'};
+
+ my $ltsm_usefsd = $temp_args_href->{'Parallel'}->{'LTSM_USEFSD'};
+ my $ltsm_fsd_servers = $temp_args_href->{'Parallel'}->{'LTSM_FSD_SERVERS'};
+ my @ltsm_fsd_server_list = split(/\s+/, $ltsm_fsd_servers);
+ my $ltsm_fsd_ports = $temp_args_href->{'Parallel'}->{'LTSM_FSD_PORT'};
+ my @ltsm_fsd_port_list = split(/\s+/, $ltsm_fsd_ports);
+
+ #my $ltsm_fsd_port = $temp_args_href->{'Parallel'}->{'LTSM_FSD_PORT'};
+
+
+ #- EPICS Controled
+ my $epics_ctrl = $temp_args_href->{'Parallel'}->{'EPICS_CTRL'};
+
+ my @epics_list = split(/\s+/, $epics_ctrl);
+
+ #- Logging the output of EBs
+ my $eb_log = $temp_args_href->{'Parallel'}->{'EB_LOG'};
+ my $eb_debug = $temp_args_href->{'Parallel'}->{'EB_DEBUG'};
+ my $nm_log = $temp_args_href->{'Parallel'}->{'NM_LOG'};
+ my @eblog_list = split(/\s+/, $eb_log);
+ my @ebdbg_list = split(/\s+/, $eb_debug);
+ my @nmlog_list = split(/\s+/, $nm_log);
+
+
+ #- Write to disk
+ my $write2disk = $temp_args_href->{'Parallel'}->{'WRITE_TO_DISK'};
+ my @write2disk_list = split(/\s+/, $write2disk);
+
+ #--- Read GbE configuration
+ my %eb_ids_gbe_hash;
+ my $eb_ids_gbe_href = \%eb_ids_gbe_hash;
+
+ &getGbEconfig($eb_ids_gbe_href);
+
+ #--- Loop over all EB processes
+ #print Dumper $eb_ids_gbe_href;
+ #exit;
+ #foreach my $ebproc ( sort keys %{$eb_ids_gbe_href} ){
+ foreach my $ebproc ( 0 .. $#bnetservers){
+
+ #- If there was a list of EBs provided via command line options
+ # go to the next $ebproc if the current $ebproc is not in this list.
+ #print "active EBs:\n";
+ #print Dumper $active_EBs_aref;
+
+ if(@$active_EBs_aref){
+ next unless( any {$_ == $ebproc} @$active_EBs_aref ); #from command line args
+ }
+ else{
+ next unless( $eb_list[$ebproc] ); #from eb.conf
+ }
+
+
+ #my $eb_ip = $eb_ids_gbe_href->{$ebproc}->{'IP'};
+
+ #- Save IP needed by other function to stop EBs.
+ #push(@EB_IP_list, $eb_ip) unless( any {$_ eq $eb_ip} @EB_IP_list );
+
+ #- Some checks on number of EB processes
+ die "Number of EB processes exceeds the number in RFIO setting! Exit." if($ebproc > $#rfio_list);
+ die "Number of EB processes exceeds the number in EPICS_CTRL setting! Exit." if($ebproc > $#epics_list);
+
+ #- Here we can overwrite default rfio settings with individual settings per EB processes
+ my $procname = sprintf("EB_PROC_%d", 1+$ebproc);
+ # $rfio_iCopyMode = $temp_args_href->{$procname}->{'RFIO_iCopyMode'};
+
+ #$href->{$ebproc}->{'IP'} = $eb_ip;
+ $href->{$ebproc}->{'IP'} = $bnetservers[$ebproc];
+ $href->{$ebproc}->{'EBNUM'} = $ebproc+1;
+ $href->{$ebproc}->{'BASEPORT'} = $base_port;
+ $href->{$ebproc}->{'PORT_LIST'} = $eb_ids_gbe_href->{$ebproc}->{'port_list'};
+ $href->{$ebproc}->{'SOURCENUM'} = scalar @{$eb_ids_gbe_href->{$ebproc}->{'port_list'}};
+
+ # JAM2016: bnet requieres udp destination nodes in a list like the ports:
+ $href->{$ebproc}->{'IP_LIST'} = $eb_ids_gbe_href->{$ebproc}->{'ip_list'};
+
+ # JAM2018: bnet requieres calibration modes in a list like the ports:
+ $href->{$ebproc}->{'CALIB_LIST'} = $eb_ids_gbe_href->{$ebproc}->{'calib_list'};
+
+ # JAM2018: bnet requires hub addresses/subevents in a list like the ports:
+ $href->{$ebproc}->{'HUB_LIST'} = $eb_ids_gbe_href->{$ebproc}->{'addr_list'};
+
+
+ $href->{$ebproc}->{'BUFSIZE_LIST'} = $eb_ids_gbe_href->{$ebproc}->{'bufsize_list'};
+ $href->{$ebproc}->{'PREFIX'} = $prefix;
+ $href->{$ebproc}->{'QUEUESIZE'} = $queuesize;
+ $href->{$ebproc}->{'MULTIDISK'} = $multidisk;
+ $href->{$ebproc}->{'FILESIZE'} = $filesize;
+
+ if( defined $temp_args_href->{$procname}->{'MULTIDISK'} ){
+ $href->{$ebproc}->{'MULTIDISK'} = $temp_args_href->{$procname}->{'MULTIDISK'};
+ }
+ elsif($multidisk){
+ $href->{$ebproc}->{'MULTIDISK'} = $href->{$ebproc}->{'EBNUM'};
+ }
+ else{
+ $href->{$ebproc}->{'MULTIDISK'} = $multidisk;
+ }
+
+ if( defined $temp_args_href->{$procname}->{'RESDOWNSCALE'} ){
+ $href->{$ebproc}->{'RESDOWNSCALE'} = $temp_args_href->{$procname}->{'RESDOWNSCALE'};
+ $href->{$ebproc}->{'RESNUMEVENTS'} = $temp_args_href->{$procname}->{'RESNUMEVENTS'};
+ $href->{$ebproc}->{'RESPATH'} = $temp_args_href->{$procname}->{'RESPATH'};
+ $href->{$ebproc}->{'RESSIZELIMIT'} = $temp_args_href->{$procname}->{'RESSIZELIMIT'};
+ }
+
+ if( defined $temp_args_href->{$procname}->{'ONLINESERVER'} ){
+ if($opt_online eq "on"){
+ $href->{$ebproc}->{'ONLINESERVER'} = "on";
+ }
+ elsif($opt_online eq "off"){
+ $href->{$ebproc}->{'ONLINESERVER'} = "off";
+ }
+ else{
+ $href->{$ebproc}->{'ONLINESERVER'} = $temp_args_href->{$procname}->{'ONLINESERVER'};
+ }
+ }
+ else{
+ $href->{$ebproc}->{'ONLINESERVER'} = "off";
+ }
+
+ $href->{$ebproc}->{'RFIO'} = $rfio_list[$ebproc] if(lc($opt_rfio) eq 'undef'); # 0|1
+ $href->{$ebproc}->{'RFIO'} = 1 if(lc($opt_rfio) eq 'on'); # 0|1
+ $href->{$ebproc}->{'RFIO'} = 0 if(lc($opt_rfio) eq 'off'); # 0|1
+ $href->{$ebproc}->{'RFIO_PATH'} = $rfio_path;
+ $href->{$ebproc}->{'RFIO_pcOptions'} = $rfio_pcOptions;
+ $href->{$ebproc}->{'RFIO_iCopyMode'} = $rfio_iCopyMode;
+ $href->{$ebproc}->{'RFIO_pcCopyPath'} = $rfio_pcCopyPath;
+ $href->{$ebproc}->{'RFIO_iCopyFrac'} = $rfio_iCopyFrac;
+ $href->{$ebproc}->{'RFIO_iMaxFile'} = $rfio_iMaxFile;
+ $href->{$ebproc}->{'RFIO_iPathConv'} = $rfio_iPathConv;
+
+
+
+ $href->{$ebproc}->{'LTSM'} = $ltsm_list[$ebproc]; # 0|1
+ $href->{$ebproc}->{'LTSM_PATH'} = $ltsm_path;
+ $href->{$ebproc}->{'LTSM_Server'} = $ltsm_server;
+ $href->{$ebproc}->{'LTSM_Node'} = $ltsm_node;
+ $href->{$ebproc}->{'LTSM_Passwd'} = $ltsm_passwd;
+ $href->{$ebproc}->{'LTSM_Filesystem'} = $ltsm_filesys;
+
+ $href->{$ebproc}->{'LTSM_USEFSD'} = $ltsm_usefsd;
+ #$href->{$ebproc}->{'LTSM_FSD_PORT'} = $ltsm_fsd_port;
+ $href->{$ebproc}->{'LTSM_FSD_PORT'} = $ltsm_fsd_port_list[$ebproc];
+ $href->{$ebproc}->{'LTSM_FSD_SERVER'} = $ltsm_fsd_server_list[$ebproc];
+
+
+
+ $href->{$ebproc}->{'EPICS_CTRL'} = $epics_list[$ebproc]; # 0|1
+
+ $href->{$ebproc}->{'DABC'} = $dabc_list[$ebproc]; # 0|1
+
+ $href->{$ebproc}->{'EB_DEBUG'} = $ebdbg_list[$ebproc]; # 0|1
+
+ $href->{$ebproc}->{'EB_LOG'} = $eblog_list[$ebproc]; # 0|1
+ $href->{$ebproc}->{'NM_LOG'} = $nmlog_list[$ebproc]; # 0|1
+
+
+ if($ebproc<5)
+ {
+ # note that for bnet setup, index does not mean eb number, but machine number!
+ # we misuse this here to save complexity of setup
+ $href->{$ebproc}->{'BNET_INP'} = $bnet_in_list[$ebproc]; # 0|1|2...
+ $href->{$ebproc}->{'BNET_BLD'} = $bnet_bld_list[$ebproc]; # 0|1|2|3
+ #print "dump bnet_bld\n";
+ #print Dumper \$bnet_bld_list[$ebproc];
+ }
+
+
+ if( $write2disk_list[$ebproc] && lc($opt_disk) eq 'undef' ){
+ if(&isVarDefined($temp_args_href->{$procname}->{'OUTDIR'}, "OUTDIR for $procname")){
+ $href->{$ebproc}->{'OUTDIR'} = $temp_args_href->{$procname}->{'OUTDIR'};
+ }
+ }
+ elsif( lc($opt_disk) eq 'on' ){
+ if(&isVarDefined($temp_args_href->{$procname}->{'OUTDIR'}, "OUTDIR for $procname")){
+ $href->{$ebproc}->{'OUTDIR'} = $temp_args_href->{$procname}->{'OUTDIR'};
+ }
+ }
+ elsif( lc($opt_disk) eq 'off' ){
+ #- do not do anything. If $href->{$ebproc}->{'OUTDIR'} is undefined,
+ # the data will go to /dev/null
+ }
+ }
+
+ $numOfEBProcs = $ebproc;
+ #print "result\n";
+ #print Dumper $ebproc;
+ # exit;
+}
+
+sub isVarDefined()
+{
+ my ($var, $msg) = @_;
+
+ my $retval = 1;
+
+ unless( defined $var ){
+ print "Undefined variable found: $msg\n";
+ $retval = 0;
+ }
+
+ return $retval;
+}
+
+sub getVarSizeArg()
+{
+ my ($ebproc) = @_;
+
+ my $i = 0;
+ my $arg = " ";
+
+ foreach my $size (@{$EB_Args_href->{$ebproc}->{'BUFSIZE_LIST'}}){
+
+ if($EB_Args_href->{$ebproc}->{'BUFSIZE_LIST'}->[$i] ==
+ $EB_Args_href->{$ebproc}->{'QUEUESIZE'}){
+ $i++;
+ next;
+ }
+
+ $arg = $arg . " -Q " . $i . ":" . $EB_Args_href->{$ebproc}->{'BUFSIZE_LIST'}->[$i];
+ $i++;
+ }
+
+ return $arg;
+}
+
+
+sub startBnet()
+{
+# here we launch the dabc bnet.
+# parameters in eb.conf can specify how many input and builder processes run on each node.
+# we misuse daq gbe setup for EB 15 to specify ports and destination nodes.
+my (@process_list);
+# setup for first EB
+my $ebproc =0; # setup for first EB will define the BNET
+ my $username = "hadaq";
+ my $dabclogin = ". /home/hadaq/soft/dabc/bin/dabclogin.head; ";
+# here test special installations:
+ my $cdworkdir = "cd /home/hadaq/oper;";
+
+ my $cmd_dabc = "/home/hadaq/soft/dabc/bin/dabc_exe.head ";
+# my $cmd_dabc = "/usr/bin/gdb -x /home/hadaq/soft/dabc/head/base/run/gdbcmd.txt --args /home/hadaq/soft/dabc/bin/dabc_exe.head ";
+
+
+ my $conf_bnet_inp = " BnetInputHades.head.xml";
+ #my $conf_bnet_bld = " BnetBuilderHades.head.xml";
+ my $conf_bnet_bld = " BnetBuilderHadesFSD.head.xml";
+
+# my @bnetservers = ("192.168.100.8","192.168.100.9","192.168.100.10","192.168.105.11", "192.168.105.12");
+
+ # variable master node now:
+ my $bnetmaster = "lxhadeb07";
+ #lxhadeb12";
+
+# before we start inidividual bnet processes, need to evaluate list of ports and nodes:
+# BNETSENDERS=[localhost:12501,localhost:12502]
+# BNETRECEIVERS= [localhost:12101,localhost:12102]
+# HADAQPORTS =[50000,50001,50002]
+
+my $bnetsenders = "[";
+my $bnetrcvs = "[";
+my @bnet_port_list = ();
+my @bnet_calib_list = ();
+my @bnet_subevt_list = ();
+my $firstsnd = 1;
+my $firstrcv = 1;
+
+ my $maxbuildservers = 4; #exclude lxhadeb12 from data writing
+ my $maxbuilders=1;
+ if ($opt_bnet>0 && $opt_bnet <16) {
+ $maxbuilders=$opt_bnet;
+ }
+
+ print "Using $maxbuilders event builder nodes from option -b $opt_bnet\n";
+my $totalbuilders=0;
+my @localbuilders = (0,0,0,0,0);
+while($totalbuilders < $maxbuilders)
+{
+ my $lasttotalbuilders=$totalbuilders;
+ for ( my $ebserver=0; $ebserver<$maxbuildservers; $ebserver=$ebserver+1){
+ my $bnet_numbuilders = $EB_Args_href->{$ebserver}->{'BNET_BLD'};
+ #print "numbuilders for $ebserver is $bnet_numbuilders \n";
+ if($totalbuilders < $maxbuilders)
+ {
+ if($localbuilders[$ebserver] < $bnet_numbuilders)
+ {
+ $localbuilders[$ebserver]++;
+ $totalbuilders++;
+ # print "Added 1 builder on EB server: $ebserver\n";
+ }
+ }
+ }
+ if ($totalbuilders==$lasttotalbuilders){ last;} # more ebs demanded than configured
+}
+
+ for ( my $ebserver=0; $ebserver<5; $ebserver=$ebserver+1){
+ print "Found $localbuilders[$ebserver] builders on EB server: $ebserver\n";
+ }
+
+my $rcvport = 12100;
+for ( my $ebserver=0; $ebserver<5; $ebserver=$ebserver+1){
+ print "Gathering processes at EB server: $ebserver\n";
+ my $sendport = 12501;
+ my $ip = $bnetservers[$ebserver];
+
+
+ # JAM2018 - need to transform HADES VLAN IPs into GSI VLAN for BNET!
+ my $gsi_ip = getIP_hades2gsi($ip);
+
+ # array of BNET values is already indexed with server id:
+ my $bnet_numsenders = $EB_Args_href->{$ebserver}->{'BNET_INP'};
+
+ for (my $six=0; $six<$bnet_numsenders; $six=$six+1)
+ {
+ $bnetsenders=$bnetsenders . "," unless ($firstsnd>0);
+ $bnetsenders=$bnetsenders . $gsi_ip.":". $sendport;
+ $sendport=$sendport+1;
+ $firstsnd=0 if($firstsnd>0);
+ }
+
+
+ for (my $rix=0; $rix<$localbuilders[$ebserver]; $rix=$rix+1)
+ {
+ $bnetrcvs=$bnetrcvs . "," unless ($firstrcv>0);
+ $bnetrcvs=$bnetrcvs . $gsi_ip.":". $rcvport;
+ $rcvport=$rcvport+1;
+ $firstrcv=0 if($firstrcv>0);
+ }
+ my $hadaqports = "[";
+ my $calibflags = "[";
+ my $subevents = "[";
+ my $firstport = 1;
+
+
+ #- add ports: note that we only use eb 1 setup and do check which ports belong to our eb server:
+ my $ix =0;
+ foreach my $port (@{$EB_Args_href->{$ebproc}->{'PORT_LIST'}}){
+ # here we only gather such ports that are assigned to our node:
+ # todo: how to distribute the ports to more than one bnet input process per server?
+ my $cflag = $EB_Args_href->{$ebproc}->{'CALIB_LIST'}[$ix];
+ my $hub = $EB_Args_href->{$ebproc}->{'HUB_LIST'}[$ix];
+ #print "ip" . $ip . " with port:" . $port ." index:" . $ix . " ip: ". $EB_Args_href->{$ebproc}->{'IP_LIST'}[$ix] . " calib: ". $cflag ." hub: ". $hub ."\n" ;
+ if($ip eq $EB_Args_href->{$ebproc}->{'IP_LIST'}[$ix])
+ {
+ $hadaqports=$hadaqports . "," unless ($firstport>0);
+ $hadaqports = $hadaqports . $port;
+ $calibflags=$calibflags . "," unless ($firstport>0);
+ $calibflags = $calibflags . $cflag;
+ $subevents=$subevents . "," unless ($firstport>0);
+ $subevents = $subevents . $hub;
+ $firstport=0 if($firstport>0);
+ }
+ $ix++;
+
+ }
+ $hadaqports=$hadaqports . "]";
+ push(@bnet_port_list, $hadaqports); # ports are per server
+
+ $calibflags=$calibflags . "]";
+ push(@bnet_calib_list, $calibflags); # flags per server
+
+ $subevents=$subevents . "]";
+ push(@bnet_subevt_list, $subevents); # flags per server
+
+
+
+ # print "node ". $gsi_ip . " uses ports ".$hadaqports . ", calibflags " .$calibflags . " hubs: ". $subevents ."\n";
+}
+$bnetsenders = $bnetsenders . "]";
+$bnetrcvs = $bnetrcvs . "]";
+
+print "bnetsenders: ". $bnetsenders ."\n";
+print "bnetreceivers: ". $bnetrcvs ."\n";
+
+ my $portid=0; #
+ my $sendid=0;
+
+ my $bnebport=12100;
+for ( my $ebserver=0; $ebserver<5; $ebserver=$ebserver+1){
+ print "Starting input processes on EB server: $ebserver\n";
+ my $ebid=$ebserver + 1; # still need unique eventbuilder ids on cluster because of epics!
+
+ my $cpu = $bnetservers[$ebserver];
+
+ my $gsi_cpu = getIP_hades2gsi($cpu);
+ # in the following, the port and ip setup of the bnet is taken from ebproc 15 or 0 only!
+
+ my $bnet_numsenders = $EB_Args_href->{$ebserver}->{'BNET_INP'};
+ #my $bnet_numbuilders = $EB_Args_href->{$ebserver}->{'BNET_BLD'};
+ print "found $bnet_numsenders senders on node $cpu \n";
+
+
+
+ my $bninpport=12501;
+
+ # loop over senders on this node and start them:
+ for(my $sender=0; $sender<$bnet_numsenders; $sender=$sender+1)
+ {
+
+ #my $sendnum= $sender + 1;
+
+ my $exports = " export MYHOST=" . $gsi_cpu . ";" .
+ " export BNETMASTER=" . $bnetmaster . ";" .
+ #" export BNINPNUM=" . $sendnum . ";" . # need to use different sender numbers on differentnodes
+ " export BNINPNUM=" . $ebid . ";" . # todo: extend if more than one sender per node
+ " export BNINPID=" . $sendid . "; " .
+ " export BNINPPORT=" . $bninpport . "; " .
+ " export BNETSENDERS=" . $bnetsenders . ";" .
+ " export BNETRECEIVERS=" . $bnetrcvs . ";" .
+ " export HADAQPORTS=" . $bnet_port_list[$ebserver] .";" .
+ " export SUBEVENTS=" . $bnet_subevt_list[$ebserver] .";" .
+ " export CALIBFLAGS=" . $bnet_calib_list[$ebserver] .";";
+
+ # todo: how to configure situation with more than one bnet input per node? hadaqports must be distributed on them...
+ #
+
+ #my $core_nr = &getCoreNr($cpu) . "," . &getCoreNr($cpu);
+ my $core_nr = &getCoreNr($cpu, 'input');
+ my $exe_dabc = "ssh -n $cpu -l $username \"$dabclogin $cdworkdir $exports taskset -c $core_nr $cmd_dabc $conf_bnet_inp 1</dev/null &\"";
+ # my $exe_dabc = "ssh -n $cpu -l $username \"$dabclogin $cdworkdir $exports $cmd_dabc $conf_bnet_inp >senderlog_$cpu.log 2>&1 &\"";
+
+
+ my $log = $log_path . "/log_" . $ebserver . "_" . "startBnetInp_". $sender. ".txt";
+ #my $log = "/dev/null 2>&1";
+
+ print "Forking:" . $exe_dabc ."\n";
+ forkMe($exe_dabc, $log, \@process_list) unless($opt_test);
+
+ $sendid = $sendid +1;
+ $bninpport = $bninpport +1;
+
+ } # bnet sender/input processes
+
+ } # end senders
+ # todo: loop over builders
+
+ for ( my $ebserver=0; $ebserver<4; $ebserver=$ebserver+1){
+ print "Starting builder processes on EB server: $ebserver\n";
+ my $ebid=$ebserver + 1; # still need unique eventbuilder ids on cluster because of epics!
+
+ my $cpu = $bnetservers[$ebserver];
+ my $gsi_cpu = getIP_hades2gsi($cpu);
+ # in the following, the port and ip setup of the bnet is taken from ebproc 15 or 0 only!
+
+ #my $bnet_numbuilders = $EB_Args_href->{$ebserver}->{'BNET_BLD'};
+
+ print "start $localbuilders[$ebserver] builders on node $cpu \n";
+
+
+
+ for(my $builder=0; $builder<$localbuilders[$ebserver]; $builder=$builder+1)
+ {
+
+ my $exports = " export MYHOST=" . $gsi_cpu . ";" .
+ " export BNETMASTER=" . $bnetmaster . ";" .
+ " export BNEBNUM=" . $ebid . ";" .
+ " export BNEBID=" . $portid . "; " .
+ " export BNEBPORT=" . $bnebport . "; " .
+ # ignore prefix at startup. always start without files and show it:
+ #" export PREFIX=" . $EB_Args_href->{$ebproc}->{'PREFIX'}. "; " .
+ " export PREFIX=--;" .
+ " export BNETSENDERS=" . $bnetsenders . ";" .
+ " export BNETRECEIVERS=" . $bnetrcvs . ";" .
+ " export HADAQPORTS=" . $bnet_port_list[$ebserver]. "; " ;
+
+
+#
+
+
+
+ if($EB_Args_href->{$ebproc}->{'OUTDIR'} ){
+ # no daqdisk demon anymore for raid6-
+ $exports = $exports . "export DAQDISK=0; export OUTDIR=/data01/data; ";
+
+# if($EB_Args_href->{$ebproc}->{'MULTIDISK'}){
+# $exports = $exports . "export DAQDISK=1; export OUTDIR=/data01; ";
+# }
+# else{
+# $exports = $exports . "export DAQDISK=0; export OUTDIR=" . $EB_Args_href->{$ebproc}->{'OUTDIR'} .";";
+# }
+
+
+
+
+# }
+ if( $EB_Args_href->{$ebproc}->{'LTSM'} ){
+
+ # switch on by number of outputs
+ $exports = $exports . " export FILEOUTPUTS=3;";
+ # additional exports for LTSM
+
+ $exports = $exports . " export LTSMPATH=". $EB_Args_href->{$ebproc}->{'LTSM_PATH'} . ";";
+ $exports = $exports . " export LTSMSERVER=". $EB_Args_href->{$ebproc}->{'LTSM_Server'} . ";";
+ $exports = $exports . " export LTSMNODE=". $EB_Args_href->{$ebproc}->{'LTSM_Node'} . ";";
+ $exports = $exports . " export LTSMPASSWD=". $EB_Args_href->{$ebproc}->{'LTSM_Passwd'} . ";";
+ $exports = $exports . " export LTSMFSNAME=". $EB_Args_href->{$ebproc}->{'LTSM_Filesystem'} . ";";
+ $exports = $exports . " export DSM_LOG=/home/hadaq/oper/ltsm". $ebid .".log;";
+
+ # new for file system daemon JAM2020:
+ if ($EB_Args_href->{$ebproc}->{'LTSM_USEFSD'}){
+ $exports = $exports . " export USEFSD=true;";
+ $exports = $exports . " export LTSMSESSIONFILES=999999;";
+ # increase number of session files if fsd is used:
+ }
+ else
+ {
+ $exports = $exports . " export USEFSD=false;";
+ $exports = $exports . " export LTSMSESSIONFILES=10;"
+ }
+
+# $exports = $exports . " export FSDPORT=". $EB_Args_href->{$ebproc}->{'LTSM_FSD_PORT'} . ";";
+# $exports = $exports . " export FSDSERVER=". $EB_Args_href->{$ebproc}->{'LTSM_FSD_SERVER'} . ";";
+
+ my $fsdix=$ebserver % 3; # number of FSD servers used TODO get from configuration
+ # in this case, server 4 will use first fsd
+ $exports = $exports . " export FSDPORT=". $EB_Args_href->{$fsdix}->{'LTSM_FSD_PORT'} . ";";
+ $exports = $exports . " export FSDSERVER=". $EB_Args_href->{$fsdix}->{'LTSM_FSD_SERVER'} . ";";
+
+
+ }
+
+ else
+ {
+ # no rfio, just local file
+ $exports = $exports . " export FILEOUTPUTS=2;";
+ }
+
+
+
+
+ } #outdir
+ else{
+ $exports = $exports . " export FILEOUTPUTS=1;";
+ # no output except for the stream server...
+ }
+
+ #my $core_nr = &getCoreNr($cpu) . "," . &getCoreNr($cpu);
+ my $core_nr = &getCoreNr($cpu, 'build');
+
+ my $exe_dabc = "ssh -n $cpu -l $username \"$dabclogin $cdworkdir $exports taskset -c $core_nr $cmd_dabc $conf_bnet_bld 1</dev/null &\"";
+ #my $exe_dabc = "ssh -n $cpu -l $username \"$dabclogin $cdworkdir $exports $cmd_dabc $conf_bnet_bld >builderlog_$cpu.log 2>&1 &\"";
+
+
+ my $log = $log_path . "/log_" . $ebserver . "_" . "startBnetBld_". $builder . ".txt";
+ #my $log = "/dev/null 2>&1";
+
+ print "Forking:" . $exe_dabc ."\n";
+ forkMe($exe_dabc, $log, \@process_list) unless($opt_test);
+
+ $ebid = $ebid + 4 ; # increment ebnum by 4 per ebserver to re-use EPICS iocs
+ $portid = $portid + 1;
+ $bnebport = $bnebport +1;
+ #
+ } # builder processes
+
+ } # servers
+
+ # finally, we need to set eb lut on cts for setup of EB1 => bnet distribution
+ #trbcmd w 0x0003 0xa0f0 0x8000`;
+ #`trbcmd w 0x0003 0xa0f0 0x1`;
+ `trbcmd setbit 0x0003 0xa0f0 0x1`;
+ # use EB1 as pseude for BNET now
+ # all calibration triggers also assigned to pseudo EB1=> bnet distribution for the moment
+ #trbcmd w 0x0003 0xa0f3 0xfff`;
+ # `trbcmd w 0x0003 0xa0f3 0xff0`;
+ `trbcmd setbit 0x0003 0xa0f3 0xff0`;
+ # use EB1 as pseudo receiver for calib triggers
+
+sleep (5); # seems also to be fine: michael, 2019-02-18
+
+# sleep (20) # old # need to wait until forking is done, otherwise it does not work via gui control xterm
+
+}
+
+
+
+sub startEvtBuilders()
+{
+ if( $EB_Args_href->{0}->{'BNET_INP'} ){
+ print "Starting Builder network...\n";
+ startBnet();
+ return;
+ }
+
+# print "DISABLING regular eventbuilder start for testing!\n";
+# return;
+########################################
+ my $username = "hadaq";
+
+ my (@process_list);
+
+ foreach my $ebproc (sort {$a <=> $b} keys %$EB_Args_href){
+
+ my $ebnum2print = $ebproc+1;
+ print "EB process: $ebnum2print\n";
+
+if($ebproc==15)
+{
+ print "Ignore eventbuilder 16, setup is reserved for BNET. \n";
+ return;
+}
+
+
+# JAM first test if we should activate dabc eventbuilder or old one
+
+ if( $EB_Args_href->{$ebproc}->{'DABC'} ){
+ print "Starting DABC process..\n";
+
+#". /home/joern/dabcwork/head/dabclogin;cd /home/joern/dabcwork/head/plugins/hadaq/app; export EBNUM=1; export STREAMS=5; export UDP00=10101; export UDP01=10102; export UDP02=10103; export UDP03=10104; export UDP04=10105 export PREFIX=be; /home/joern/dabcwork/head/bin/dabc_exe EventBuilderHades.xml &" > /dev/null 2>&1 &
+
+ my $cpu = $EB_Args_href->{$ebproc}->{'IP'};
+# JAM old, direct to version
+ #my $dabclogin = ". /home/hadaq/soft/dabc/head/dabclogin;";
+# JAM default:
+#my $dabclogin = ". /home/hadaq/soft/dabc/bin/dabclogin;";#
+#my $dabclogin = ". /home/hadaq/soft/dabc/bin/dabclogin.275;";
+ my $dabclogin = ". /home/hadaq/soft/dabc/bin/dabclogin.head; ";
+# here test special installations:
+ my $cdworkdir = "cd //home/hadaq/oper;";
+
+# JAM old, direct to version
+#my $cmd_dabc = "/home/hadaq/soft/dabc/head/bin/dabc_exe ";
+# JAM default:
+#my $cmd_dabc = "/home/hadaq/soft/dabc/bin/dabc_exe ";
+# here test special installations:
+# my $cmd_dabc = "/home/hadaq/soft/dabc/bin/dabc_exe.275 ";
+my $cmd_dabc = "/home/hadaq/soft/dabc/bin/dabc_exe.head ";
+
+# my $conf_dabc = " EventBuilderHades.xml";
+# my $conf_dabc = " EventBuilderHades.275.xml";
+ my $conf_dabc = " EventBuilderHades.head.xml";
+
+ my $exports = " export LC_ALL=C;" .
+ " export EBNUM=" . $EB_Args_href->{$ebproc}->{'EBNUM'} . "; " .
+ " export STREAMS=" . $EB_Args_href->{$ebproc}->{'SOURCENUM'} . "; " .
+ " export PREFIX=" . $EB_Args_href->{$ebproc}->{'PREFIX'}. "; " ;
+
+ my @port_list = ();
+
+ #- add ports
+ my $ix =0;
+ foreach my $port (@{$EB_Args_href->{$ebproc}->{'PORT_LIST'}}){
+ #$cmd_nm = $cmd_nm . " -i UDP:0.0.0.0:" . $port;
+ my $index=sprintf("%02d", $ix++);
+ $exports = $exports . " export UDP". $index. "=" . $port . "; ";
+ push(@port_list, $port);
+ }
+ &cpPortList2EB(\@port_list, $EB_Args_href->{$ebproc}->{'EBNUM'}, $cpu);
+
+
+
+
+
+
+# MULTIDISK
+
+#- add output type
+
+ if($EB_Args_href->{$ebproc}->{'OUTDIR'} ){
+ if($EB_Args_href->{$ebproc}->{'MULTIDISK'}){
+ $exports = $exports . "export DAQDISK=1; export OUTDIR=/data01; ";
+ }
+ else{
+ $exports = $exports . "export DAQDISK=0; export OUTDIR=" . $EB_Args_href->{$ebproc}->{'OUTDIR'} .";";
+ }
+
+ if( $EB_Args_href->{$ebproc}->{'LTSM'} ){
+
+ $exports = $exports . " export FILEOUTPUTS=3;";
+ # additional exports for LTSM
+
+ $exports = $exports . " export LTSMPATH=". $EB_Args_href->{$ebproc}->{'LTSM_PATH'} . ";";
+ $exports = $exports . " export LTSMSERVER=". $EB_Args_href->{$ebproc}->{'LTSM_Server'} . ";";
+ $exports = $exports . " export LTSMNODE=". $EB_Args_href->{$ebproc}->{'LTSM_Node'} . ";";
+ $exports = $exports . " export LTSMPASSWD=". $EB_Args_href->{$ebproc}->{'LTSM_Passwd'} . ";";
+ $exports = $exports . " export LTSMFSNAME=". $EB_Args_href->{$ebproc}->{'LTSM_Filesystem'} . ";";
+
+# switch on by number of outputs
+ }
+################## deprecated, keep code for optional testing?
+# JAM 5-2017 - we never run rfio and ltsm in parallel.
+# if( $EB_Args_href->{$ebproc}->{'RFIO'} ){
+#
+# $exports = $exports . " export FILEOUTPUTS=3;";
+# # additional exports for RFIO
+#
+# $exports = $exports . " export RFIOPATH=". $EB_Args_href->{$ebproc}->{'RFIO_PATH'} . ";";
+# $exports = $exports . " export RFIOLUSTREPATH=". $EB_Args_href->{$ebproc}->{'RFIO_pcCopyPath'} . ";";
+# $exports = $exports . " export RFIOCOPYMODE=". $EB_Args_href->{$ebproc}->{'RFIO_iCopyMode'} . ";";
+# $exports = $exports . " export RFIOCOPYFRAC=". $EB_Args_href->{$ebproc}->{'RFIO_iCopyFrac'} . ";";
+# $exports = $exports . " export RFIOMAXFILE=". $EB_Args_href->{$ebproc}->{'RFIO_iMaxFile'} . ";";
+# $exports = $exports . " export RFIOPATHCONV=". $EB_Args_href->{$ebproc}->{'RFIO_iPathConv'} . ";";
+#
+# # switch on by number of outputs
+# }
+#######################################
+ else
+ {
+ # no rfio, just local file
+ $exports = $exports . " export FILEOUTPUTS=2;";
+ }
+
+
+
+
+ } #outdir
+ else{
+ $exports = $exports . " export FILEOUTPUTS=1;";
+ # no output except for the stream server...
+ }
+
+
+
+
+
+
+
+
+
+# EPICSCONTROL ? always enabled for production
+# SMALLFILES for online monitoring node
+
+# Jul14 beamtime setup 3 cores for dabc
+ #my $core_nr = &getCoreNr($cpu) . "," . &getCoreNr($cpu) . "," . &getCoreNr($cpu);
+
+# try 2 cores each dabc for more dabc nodes:
+ my $core_nr = &getCoreNr($cpu) . "," . &getCoreNr($cpu);
+# my $core_nr = &getCoreNr($cpu);
+# dabc is set to 3 cores
+
+# JAM use fixed core number for kp1pc092 tests:
+# my $core_nr = 1;
+ my $exe_dabc = "ssh -n $cpu -l $username \"$dabclogin $cdworkdir $exports taskset -c $core_nr $cmd_dabc $conf_dabc 1</dev/null &\"";
+# my $exe_dabc = "ssh -n $cpu -l $username \"$dabclogin $cdworkdir $exports $cmd_dabc $conf_dabc &\"";
+
+
+ my $log = $log_path . "/log_" . $ebproc . "_" . "startEB.txt";
+ #my $log = "/dev/null 2>&1";
+
+ print "Forking:" . $exe_dabc ."\n";
+ forkMe($exe_dabc, $log, \@process_list) unless($opt_test);
+
+}
+
+else
+{
+# the standard EB processes mode:
+ print "Starting evtbuild/netmem processes..\n";
+
+ #--- Prepare execution of daq_evtbuild
+ my $cmd_eb = "/home/hadaq/bin/daq_evtbuild" .
+ " -m " . $EB_Args_href->{$ebproc}->{'SOURCENUM'} .
+ " -q " . $EB_Args_href->{$ebproc}->{'QUEUESIZE'} .
+ " -S " . $EB_Args_href->{$ebproc}->{'EBNUM'} .
+ " --ebnum " . $EB_Args_href->{$ebproc}->{'EBNUM'} .
+ " -x " . $EB_Args_href->{$ebproc}->{'PREFIX'};
+
+ #- add queue variable size args
+ my $varsize_arg = &getVarSizeArg($ebproc);
+ $cmd_eb = $cmd_eb . $varsize_arg;
+
+ #- add output type
+ if( defined $EB_Args_href->{$ebproc}->{'OUTDIR'} ){
+ if($EB_Args_href->{$ebproc}->{'MULTIDISK'}){
+ $cmd_eb = $cmd_eb . " -d file -o " . "/data01/data";
+ }
+ else{
+ $cmd_eb = $cmd_eb . " -d file -o " . $EB_Args_href->{$ebproc}->{'OUTDIR'};
+ }
+ }
+ else{
+ $cmd_eb = $cmd_eb . " -d null";
+ }
+
+ #- add file size
+ $cmd_eb = $cmd_eb . " --filesize " . $EB_Args_href->{$ebproc}->{'FILESIZE'};
+
+ #- add second output with small hdl files
+ if( defined $EB_Args_href->{$ebproc}->{'RESDOWNSCALE'} ){
+ $cmd_eb = $cmd_eb . " --resdownscale " . $EB_Args_href->{$ebproc}->{'RESDOWNSCALE'} .
+ " --resnumevents " . $EB_Args_href->{$ebproc}->{'RESNUMEVENTS'} .
+ " --respath " . $EB_Args_href->{$ebproc}->{'RESPATH'} .
+ " --ressizelimit " . $EB_Args_href->{$ebproc}->{'RESSIZELIMIT'};
+ }
+
+ my $cpu = $EB_Args_href->{$ebproc}->{'IP'};
+
+ #- add rfio args
+ my $rfio;
+ if( $EB_Args_href->{$ebproc}->{'RFIO'} ){
+ $rfio = " --rfio rfiodaq:gstore:" . $EB_Args_href->{$ebproc}->{'RFIO_PATH'} .
+ " --rfiolustre " . $EB_Args_href->{$ebproc}->{'RFIO_pcCopyPath'} .
+ " --rfio_pcoption " . $EB_Args_href->{$ebproc}->{'RFIO_pcOptions'} .
+ " --rfio_icopymode " . $EB_Args_href->{$ebproc}->{'RFIO_iCopyMode'} .
+ " --rfio_icopyfrac " . $EB_Args_href->{$ebproc}->{'RFIO_iCopyFrac'} .
+ " --rfio_imaxfile " . $EB_Args_href->{$ebproc}->{'RFIO_iMaxFile'} .
+ " --rfio_ipathconv " . $EB_Args_href->{$ebproc}->{'RFIO_iPathConv'};
+ }
+
+ $cmd_eb = $cmd_eb . $rfio if( defined $rfio );
+
+ #- add multiple disk arg (ctrl via daq_disks)
+ if($EB_Args_href->{$ebproc}->{'MULTIDISK'} &&
+ defined $EB_Args_href->{$ebproc}->{'OUTDIR'}){
+ $cmd_eb = $cmd_eb . " --multidisk " . $EB_Args_href->{$ebproc}->{'MULTIDISK'};
+ }
+
+ #- add online RPC server
+ if( $EB_Args_href->{$ebproc}->{'ONLINESERVER'} eq "on" ){
+ $cmd_eb = $cmd_eb . " --online";
+ }
+
+ #- add epics controlled
+ $cmd_eb = $cmd_eb . " --epicsctrl " if( $EB_Args_href->{$ebproc}->{'EPICS_CTRL'} );
+
+ # switch on debug output
+ $cmd_eb = $cmd_eb . " --debug trignr --debug errbit --debug word " if( $EB_Args_href->{$ebproc}->{'EB_DEBUG'} );
+
+
+
+ #- logging the output
+ my $eblog_file = "/tmp/log_eb_" . $EB_Args_href->{$ebproc}->{'EBNUM'} . ".txt";
+ my $eb_log = "1>$eblog_file 2>$eblog_file";
+ $eb_log = "1>/dev/null 2>/dev/null" unless( $EB_Args_href->{$ebproc}->{'EB_LOG'} );
+
+ my $time = 1. * $ebproc;
+ my $sleep_cmd = "sleep " . $time;
+
+ my $core_nr = &getCoreNr($cpu);
+
+ my $exe_eb = "ssh -n $cpu -l $username \"cd /home/hadaq/oper; export DAQ_SETUP=/home/hadaq/oper/eb; taskset -c $core_nr $cmd_eb $eb_log &\"";
+
+ #print "exec: $exe_eb\n";
+
+ #--- Prepare execution of daq_netmem
+ my $cmd_nm = "/home/hadaq/bin/daq_netmem" .
+ " -m " . $EB_Args_href->{$ebproc}->{'SOURCENUM'} .
+ " -q " . $EB_Args_href->{$ebproc}->{'QUEUESIZE'} .
+ " -S " . $EB_Args_href->{$ebproc}->{'EBNUM'};
+
+ #- add queue variable size args
+ $cmd_nm = $cmd_nm . $varsize_arg;
+
+ my @port_list = ();
+
+ #- add ports
+ foreach my $port (@{$EB_Args_href->{$ebproc}->{'PORT_LIST'}}){
+ #$cmd_nm = $cmd_nm . " -i UDP:0.0.0.0:" . $port;
+ $cmd_nm = $cmd_nm . " -i " . $port;
+
+ push(@port_list, $port);
+ }
+
+ &cpPortList2EB(\@port_list, $EB_Args_href->{$ebproc}->{'EBNUM'}, $cpu);
+
+ #- logging the output
+ my $nmlog_file = "/tmp/log_nm_" . $EB_Args_href->{$ebproc}->{'EBNUM'} . ".txt";
+ my $nm_log = "1>$nmlog_file 2>$nmlog_file";
+ $nm_log = "1>/dev/null 2>/dev/null" unless( $EB_Args_href->{$ebproc}->{'NM_LOG'} );
+
+ $core_nr = &getCoreNr($cpu);
+
+ my $exe_nm = "ssh -n $cpu -l $username \"cd /home/hadaq/oper; export DAQ_SETUP=/home/hadaq/oper/eb; taskset -c $core_nr $cmd_nm $nm_log &\"";
+
+ #print "exec: $exe_nm\n";
+
+ #--- Open permissions for shared memory
+ my $eb_shmem = "daq_evtbuild" . $EB_Args_href->{$ebproc}->{'EBNUM'} . ".shm";
+ my $nm_shmem = "daq_netmem" . $EB_Args_href->{$ebproc}->{'EBNUM'} . ".shm";
+ my $exe_open_eb = "ssh -n $cpu -l $username \"chmod 775 /dev/shm/$eb_shmem\"";
+ my $exe_open_nm = "ssh -n $cpu -l $username \"chmod 775 /dev/shm/$nm_shmem\"";
+
+ &forkEB($exe_eb, $exe_nm, $exe_open_eb, $exe_open_nm, \@process_list);
+ }
+
+}
+# if dabc
+
+ #- Wait for children
+ foreach my $cur_child_pid (@process_list) {
+ waitpid($cur_child_pid,0);
+ }
+
+}
+# foreach
+
+sub stopEvtBuilders()
+{
+ my $username = "hadaq";
+
+ my @process_list = ();
+
+ #--- Loop over server IPs
+ # foreach my $ip (@EB_IP_list){
+foreach my $ip (@bnetservers){
+
+ my $exe = "ssh -n $ip -l $username \"/home/hadaq/bin/cleanup_evtbuild.pl; /home/hadaq/bin/ipcrm.pl\"";
+
+ if($opt_verb){
+ print "Killing running EBs...\n";
+ print "Exec: $exe\n";
+ }
+
+ my $log = $log_path . "/log_" . $ip . "_" . "stopEB.txt";
+
+ forkMe($exe, $log, \@process_list) unless($opt_test);
+ }
+
+ #- Wait for children
+ foreach my $cur_child_pid (@process_list) {
+ print "wait for $cur_child_pid\n";
+ waitpid($cur_child_pid,0);
+ }
+}
+
+sub cpPortList2EB()
+{
+ my ($port_list_aref, $ebnr, $cpu) = @_;
+
+ my $tmpfile = "/tmp/eb" . $ebnr . "_" . $cpu . ".txt";
+
+ #- First write ports to tmp file
+ my $fh = new FileHandle(">$tmpfile");
+
+ if(!$fh) {
+ my $txt = "\nError! Could not open file \"$tmpfile\" for output. Exit.\n";
+ print STDERR $txt;
+ print $txt;
+ exit(128);
+ }
+
+ foreach my $port (@$port_list_aref){
+ print $fh "$port\n";
+ }
+
+ $fh->close();
+
+ #- Copy this tmp file to EB
+ my $exe_cp = "scp $tmpfile hadaq\@$cpu:/tmp/ 1>/dev/null 2>/dev/null";
+ system($exe_cp);
+}
+
+sub startIOC()
+{
+ my $ioc_dir = "/home/scs/ebctrl/ioc/iocBoot/iocebctrl";
+
+ &writeIOC_stcmd( $ioc_dir );
+
+ print "Starting IOCs...\n" if($opt_verb);
+
+ foreach my $ebproc (keys %$EB_Args_href){
+
+ my $stcmd = sprintf("st_eb%02d.cmd", 1 + $ebproc);
+ my $screen_name = sprintf("ioc_eb%02d", 1 + $ebproc);
+
+ my $cmd = "bash; . /home/scs/.bashrc; export HOSTNAME=\\\$(hostname); cd $ioc_dir; screen -dmS $screen_name ../../bin/linux-x86_64/ebctrl $stcmd";
+ my $cpu = $EB_Args_href->{$ebproc}->{'IP'};
+ # JAM2016: this is kludge for bnet:
+ # first IP in hub configuration of pseude EB15 might be set differently
+ # we always reset it to match lxhadeb05 where epics for builder should belong
+ if($ebproc == 15)
+ {
+ $cpu='192.168.100.15';
+ }
+ #another dirty workaround for bnet at eb0:
+ if ($ebproc == 0) {
+ $cpu = '192.168.100.8';
+ }
+ # end bnet kludge
+ my $exe = "ssh -n $cpu -l scs \"$cmd\"";
+
+ print "Exec: $exe\n" if($opt_verb);
+ system($exe) unless($opt_test);
+ }
+}
+
+sub smallestEBProcNum()
+{
+ my $smallest = 1000;
+
+ foreach my $ebproc (keys %$EB_Args_href){
+ $smallest = $ebproc if($smallest > $ebproc);
+ }
+
+ return $smallest;
+}
+
+sub writeIOC_stcmd()
+{
+ my ($ioc_dir) = @_;
+
+ # JAM first evaluate ports for ca list
+ my $epicscalist = "192.168.111.255";
+ foreach my $ebproc (keys %$EB_Args_href){
+ $epicscalist=sprintf("%s 192.168.111.255:%d", $epicscalist, 10001 + $ebproc);
+ }
+
+ print "Copying st.cmd files to servers...\n" if($opt_verb);
+
+ my $smallest_ebproc = &smallestEBProcNum();
+
+ foreach my $ebproc (keys %$EB_Args_href){
+
+ my $ebNr = 1 + $ebproc;
+ my $ebnum = sprintf("eb%02d", $ebNr);
+ my $serverport = 10001+ $ebproc;
+
+
+ #- in MBytes
+ my $maxFileSize = $EB_Args_href->{$ebproc}->{'FILESIZE'};
+
+ my $ebtype = "slave";
+ my $comment_genrunid = "#";
+ my $comment_totalevt = "#";
+
+ if($ebproc == $smallest_ebproc){
+ $ebtype = "master";
+ $comment_genrunid = "";
+ $comment_totalevt = "";
+ }
+
+# if($ebNr == 1){
+# $comment_totalevt = "";
+# }
+
+ my $ioc_stcmd = <<EOF;
+#!../../bin/linux-x86_64/ebctrl
+
+## Set EPICS environment
+
+< envPaths
+
+epicsEnvSet(FILESIZE,"$maxFileSize")
+epicsEnvSet(EBTYPE,"$ebtype")
+epicsEnvSet(EBNUM,"$ebNr")
+epicsEnvSet(ERRBITLOG, "1")
+epicsEnvSet(ERRBITWAIT, "30")
+epicsEnvSet(EPICS_CAS_SERVER_PORT,"$serverport")
+## epicsEnvSet(EPICS_CA_ADDR_LIST,"192.168.111.255")
+epicsEnvSet(EPICS_CA_ADDR_LIST,"$epicscalist")
+epicsEnvSet(EPICS_CA_AUTO_ADDR_LIST,"NO")
+epicsEnvSet(PATH,"/home/scs/base-3-14-11/bin/linux-x86_64:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:.")
+
+cd \${TOP}
+
+## Register all support components
+dbLoadDatabase("dbd/ebctrl.dbd")
+ebctrl_registerRecordDeviceDriver(pdbbase)
+
+## Load record instances
+dbLoadRecords("db/stats.db", "PREFIX=HAD:IOC:,IOC=$ebnum")
+dbLoadRecords("db/evtbuild.db","eb=$ebnum")
+dbLoadRecords("db/netmem.db","eb=$ebnum")
+dbLoadRecords("db/errbit1.db","eb=$ebnum")
+dbLoadRecords("db/errbit2.db","eb=$ebnum")
+dbLoadRecords("db/trignr1.db","eb=$ebnum")
+dbLoadRecords("db/trignr2.db","eb=$ebnum")
+dbLoadRecords("db/portnr1.db","eb=$ebnum")
+dbLoadRecords("db/portnr2.db","eb=$ebnum")
+dbLoadRecords("db/trigtype.db","eb=$ebnum")
+## JAM disable cpu module to test epicshangup issue:
+## dbLoadRecords("db/cpu.db","eb=$ebnum")
+dbLoadRecords("db/errbitstat.db","eb=$ebnum")
+$comment_totalevt dbLoadRecords("db/totalevtstat.db")
+$comment_genrunid dbLoadRecords("db/genrunid.db","eb=$ebnum")
+
+## Set this to see messages from mySub
+var evtbuildDebug 0
+var netmemDebug 0
+var genrunidDebug 0
+var writerunidDebug 0
+var errbit1Debug 0
+var errbit2Debug 0
+var trigtypeDebug 1
+var cpuDebug 0
+var errbitstatDebug 0
+$comment_totalevt var totalevtscompDebug 0
+cd \${TOP}/iocBoot/\${IOC}
+iocInit()
+
+## Start any sequence programs
+#seq sncExample,"user=scsHost"
+
+dbl > \${TOP}/iocBoot/\${IOC}/$ebnum.dbl
+
+EOF
+
+ my $outfile = "/tmp/st_" . $ebnum . ".cmd";
+ my $fh = new FileHandle(">$outfile");
+
+ if(!$fh) {
+ my $txt = "\nError! Could not open file \"$outfile\" for output. Exit.\n";
+ print STDERR $txt;
+ print $txt;
+ exit(128);
+ }
+
+ print $fh $ioc_stcmd;
+ $fh->close();
+
+ my $ip = $EB_Args_href->{$ebproc}->{'IP'};
+ my $cmd = "scp $outfile scs\@$ip:$ioc_dir/.";
+
+ print "Exec: $cmd\n" if($opt_verb);
+ system($cmd) unless($opt_test);
+ }
+}
+
+sub killIOC()
+{
+ my %ioc;
+ my $ioc_href = \%ioc;
+
+ print "Looking for running IOCs...\n" if($opt_verb);
+
+ #--- Loop over server IPs
+
+ #foreach my $ip (@EB_IP_list){
+ foreach my $ip (@bnetservers){
+
+ &findRunningIOC($ip, $ioc_href);
+ }
+
+ #print Dumper \%$ioc_href;
+
+ &writeExpectIOC() if(%$ioc_href);
+
+ if($opt_verb){
+ print "Killing running IOCs...\n";
+ print "No IOCs found - nothing to kill, continue...\n" unless(%$ioc_href);
+ }
+
+ my (@process_list);
+
+ foreach my $ip ( %$ioc_href ){
+ foreach my $ioc ( @{$ioc_href->{$ip}} ){
+
+ my $cmd = $expect_ioc_script . " " . $ip . " " . $ioc;
+ my $log = $log_path . "/log_" . $ip . "_" . $ioc . ".txt";
+ print "cmd: $cmd\n" if($opt_verb);
+ &forkMe($cmd, $log, \@process_list);
+ }
+ }
+
+
+
+
+ #- Wait for children
+ foreach my $cur_child_pid (@process_list) {
+ waitpid($cur_child_pid,0);
+ }
+
+ ### just kill the remaining stuff
+ @process_list = ();
+
+ #foreach my $ip (@EB_IP_list){
+ foreach my $ip (@bnetservers){
+ my $cmd = qq|ssh scs\@$ip "/usr/bin/pkill -f \\"SCREEN -dmS ioc_eb\\""|;
+ print $cmd;
+ &forkMe($cmd, "/tmp/ioc_kill_$ip", \@process_list);
+ }
+
+ foreach my $cur_child_pid (@process_list) {
+ waitpid($cur_child_pid,0);
+ }
+
+ #sleep 1;
+
+ ### just kill the remaining stuff
+ @process_list = ();
+ #foreach my $ip (@EB_IP_list){
+ foreach my $ip (@bnetservers){
+ my $cmd = qq|ssh scs\@$ip "/usr/bin/pkill -9 -f \\"SCREEN -dmS ioc_eb\\""|;
+ &forkMe($cmd, "/tmp/ioc_kill2_$ip", \@process_list);
+ }
+
+ foreach my $cur_child_pid (@process_list) {
+ waitpid($cur_child_pid,0);
+ }
+
+}
+
+sub forkMe()
+{
+ my ($cmd, $log, $proc_list) = @_;
+
+ my $child = fork();
+
+ if( $child ){ # parent
+ push( @$proc_list, $child );
+ }
+ elsif( $child == 0 ) { # child
+ system("$cmd >$log 2>&1 ");
+ exit(0);
+ }
+ else{
+ print "Could not fork: $!\n";
+ exit(1);
+ }
+}
+
+sub forkEB()
+{
+ my ($exe_eb, $exe_nm, $exe_open_eb, $exe_open_nm, $proc_list) = @_;
+
+ my $child = fork();
+
+ if( $child ){ # parent
+ push( @$proc_list, $child );
+ }
+ elsif( $child == 0 ) { # child
+ #--- Execute Event Builder
+ print "Exec: $exe_eb\n" if($opt_verb);
+ system($exe_eb) unless($opt_test);
+
+ #sleep(1);
+
+ #--- Open permissions for EB shared memory
+ # ! Permissions should be opened by EB process
+ #print "Exec: $exe_open_eb\n" if($opt_verb);
+ #system($exe_open_eb) unless($opt_test);
+
+ #sleep(2);
+
+ #--- Execute Net-2-Memory
+ print "Exec: $exe_nm\n" if($opt_verb);
+ system($exe_nm) unless($opt_test);
+
+ #sleep(1);
+
+ #--- Open permissions for NM shared memory
+ # ! Permissions should be opened by EB process
+ #print "Exec: $exe_open_nm\n" if($opt_verb);
+ #system($exe_open_nm) unless($opt_test);
+
+ exit(0);
+ }
+ else{
+ print "Could not fork: $!\n";
+ exit(1);
+ }
+}
+
+sub findRunningIOC()
+{
+ my ($cpu, $ioc_href) = @_;
+
+ `ssh -n $cpu -l scs \"screen -wipe\"`;
+ my $exe = "ssh -n $cpu -l scs \"screen -ls\"";
+
+ my @output = `$exe`;
+
+ foreach my $line (@output){
+ if($line =~ /\d+\.(ioc_eb\d{2})\s+/){
+ my $name = $1;
+ push( @{$ioc_href->{$cpu}}, $name );
+ print "Found IOC: $name on $cpu\n" if($opt_verb);
+ }
+ }
+}
+
+sub writeExpectIOC()
+{
+ # This expect script can be executed to exit IOC.
+
+ #! Look if /tmp dir exists
+ my $tmp_dir = dirname("/tmp");
+ if ( !(-d $tmp_dir) ){
+ print "\nCannot access /tmp directory!\nExit.\n";
+ exit(1);
+ }
+
+ my $expect_script_my = <<EOF;
+#!/usr/bin/expect -f
+
+# This script is automatically generated by startup.pl
+# Do not edit, the changes will be lost.
+
+# Print args
+send_user "\$argv0 [lrange \$argv 0 \$argc]\\n"
+
+# Get args
+#
+# ip : IP address of the server
+# iocname : name of IOC screen process (screen -ls)
+#
+if {\$argc>0} {
+ set ip [lindex \$argv 0]
+ set iocname [lindex \$argv 1]
+} else {
+ send_user "Usage: \$argv0 ip iocname\\n"
+}
+
+spawn ssh scs@\$ip
+
+#expect {
+# "error" { exit; }
+# "login:" { exit; }
+# "Password:" { exit; }
+#}
+
+set timeout 20
+#240
+
+expect "~\$ "
+send "screen -r \$iocname\\r"
+expect "epics> "
+send "exit\\r"
+expect "~\$ "
+
+EOF
+
+ my $fh = new FileHandle(">$expect_ioc_script");
+
+ if(!$fh) {
+ my $txt = "\nError! Could not open file \"$expect_ioc_script\" for output. Exit.\n";
+ print STDERR $txt;
+ print $txt;
+ exit(128);
+ }
+
+ print $fh $expect_script_my;
+ $fh->close();
+
+ #- open permissions
+ system("chmod 755 $expect_ioc_script");
+}
+
+sub getGbEconfig()
+{
+ #
+ # Read DB configurations of GbE and CTS,
+ # look for active data sources as well as
+ # for EB IPs and ports.
+ #
+
+ my ($eb_ids_href) = @_;
+
+ my $data_sources = $temp_args_href->{'Parallel'}->{'DATA_SOURCES'};
+ my $gbe_conf = $temp_args_href->{'Parallel'}->{'GBE_CONF'};
+ #my $cts_conf = $temp_args_href->{'Parallel'}->{'CTS_CONF'};
+
+ my %activeSources_hash;
+ my $activeSources_href = \%activeSources_hash;
+
+ &readActiveSources($data_sources, $activeSources_href);
+
+ my @id_list;
+ my $id_list_aref = \@id_list;
+
+ #&readEBids($cts_conf, $id_list_aref);
+
+ #- Overwrite array with EB numbers
+ @id_list = (0 .. 15);
+ #print Dumper $id_list_aref;
+
+ &readEBports($gbe_conf, $activeSources_href, $id_list_aref, $eb_ids_href);
+}
+
+sub readEBids()
+{
+ #
+ # Read EB Ids
+ #
+
+ my ($file, $id_list_aref) = @_;
+
+ my $nnn_table = 0;
+ my $val_table = 0;
+
+ my $SPACE = "";
+
+ my $fh = new FileHandle("$file", "r");
+
+ while(<$fh>){
+
+ #- Remove all comments
+ $_ =~ s{ # Substitue...
+ \# # ...a literal octothorpe
+ [^\n]* # ...followed by any number of non-newlines
+ }
+ {$SPACE}gxms; # Raplace it with a single space
+
+ #- Skip line if it contains only whitespaces
+ next unless(/\S/);
+
+ if(/^(\s+)?!Value\stable/){
+ $val_table = 1;
+ $nnn_table = 0;
+ next;
+ }
+ elsif(/^(\s+)?!\w+/){
+ $val_table = 0;
+ $nnn_table = 1;
+ }
+
+ if($val_table){
+ my (@vals) = split(" ", $_);
+ my @id_list1 = split("", $vals[12]);
+ my @id_list2 = split("", $vals[13]);
+ foreach my $id (@id_list1){
+ push(@$id_list_aref, hex($id));
+ }
+ foreach my $id (@id_list2){
+ push(@$id_list_aref, hex($id));
+ }
+ }
+ elsif($nnn_table){
+ }
+ }
+
+ $fh->close;
+}
+
+sub readEBports()
+{
+ #
+ # Read EB IPs and ports accoring to EB Id (type)
+ # and TRB-Net addresses of active data sources.
+ #
+
+ my ($file, $activeSources_href, $id_list_aref, $ports_href) = @_;
+
+ my $nnn_table = 0;
+ my $val_table = 0;
+
+ my $fh = new FileHandle("$file", "r");
+
+ &isFileDefined($fh, $file);
+
+ my %tmp;
+ my $tmp_href = \%tmp;
+
+ my $SPACE = "";
+
+ while(<$fh>){
+
+ #print $_;
+ #- Remove all comments
+ $_ =~ s{ # Substitue...
+ \# # ...a literal octothorpe
+ [^\n]* # ...followed by any number of non-newlines
+ }
+ {$SPACE}gxms; # Raplace it with a single space
+
+ #- Skip line if it contains only whitespaces
+ next unless(/\S/);
+
+ #print $_;
+ if(/^(\s+)?!Value\stable/){
+ $val_table = 1;
+ $nnn_table = 0;
+ next;
+ }
+ elsif(/^(\s+)?!\w+/){
+ $nnn_table = 1;
+ $val_table = 0;
+ }
+
+ if($val_table){
+ my (@vals) = split(" ", $_);
+ my $id = $vals[1];
+
+ #if($id <0 or $id >15) {
+ # print "error: in $file there is a line with an eventbuilder number different than 0..15, the number given in the file is $id. please correct the config file.\n";
+ # exit(128);
+ #}
+
+
+ #- Accept only EB Ids from CTS config file
+ #print "value: $_";
+ next unless( any {$_ eq $id} @$id_list_aref );
+
+ #print Dumper \@vals;
+ #print "active sources: "; print Dumper $activeSources_href->{'addr_list'};
+ #exit;
+
+ my $ip = &getIP_hex2dec($vals[6]);
+ my $port = &getPort_hex2dec($vals[2]);
+ my $addr = $vals[0];
+ my $calib = $vals[7]; # jam2018 - add list of trb3 tdc calibration modes
+
+ # print "got: ip: $ip, port: $port, addr: $addr\n";
+ #- Accept only sources from active source list
+ if( any {hex($_) == hex($addr)} @{$activeSources_href->{'addr_list'}} ){
+ $tmp_href->{$id}->{'IP'} = $ip;
+ push( @{$tmp_href->{$id}->{'port_list'}}, $port );
+ push( @{$tmp_href->{$id}->{'addr_list'}}, $addr );
+ push( @{$tmp_href->{$id}->{'calib_list'}}, $calib );
+
+ # JAM2016: for bnet we need the receiver nodes per port as list also:
+ push( @{$tmp_href->{$id}->{'ip_list'}}, $ip );
+
+ }
+ }
+ }
+
+ $fh->close;
+
+ # print Dumper $tmp_href;
+
+ #- Sort hash according to active data source list
+ my $numids= scalar keys %tmp;
+ #print "number of ids: $numids \n";
+ foreach my $id (keys %tmp){
+ $ports_href->{$id}->{'IP'} = $tmp_href->{$id}->{'IP'};
+
+ foreach my $addr (@{$activeSources_href->{'addr_list'}}){
+
+ my $ind1 = first_index {$_ eq $addr} @{$tmp_href->{$id}->{'addr_list'}};
+ my $ind2 = first_index {$_ eq $addr} @{$activeSources_href->{'addr_list'}};
+
+ next if($ind1 == -1);
+
+# push( @{$ports_href->{$id}->{'port_list'}}, $tmp_href->{$id}->{'port_list'}->[$ind1]);
+# # added for bnet JAM:
+# push( @{$ports_href->{$id}->{'ip_list'}}, $tmp_href->{$id}->{'ip_list'}->[$ind1]);
+# push( @{$ports_href->{$id}->{'calib_list'}}, $tmp_href->{$id}->{'calib_list'}->[$ind1]);
+#
+# push( @{$ports_href->{$id}->{'addr_list'}}, $addr);
+# push( @{$ports_href->{$id}->{'bufsize_list'}}, $activeSources_href->{'bufsize_list'}->[$ind2]);
+
+
+ if($id == 0 && $numids==1){
+ # fill all ids of active inputs with setup for id 0 of bnet
+ foreach my $bid (0 .. $#bnetservers){
+ push( @{$ports_href->{$bid}->{'port_list'}}, $tmp_href->{$id}->{'port_list'}->[$ind1]);
+ push( @{$ports_href->{$bid}->{'ip_list'}}, $tmp_href->{$id}->{'ip_list'}->[$ind1]);
+ push( @{$ports_href->{$bid}->{'calib_list'}}, $tmp_href->{$id}->{'calib_list'}->[$ind1]);
+ push( @{$ports_href->{$bid}->{'addr_list'}}, $addr);
+ push( @{$ports_href->{$bid}->{'bufsize_list'}}, $activeSources_href->{'bufsize_list'}->[$ind2]);
+ }
+ }
+ else
+ {
+ #old EB mode/no bnet: config file defines all. DO WE NEED THIS STILL?
+ push( @{$ports_href->{$id}->{'port_list'}}, $tmp_href->{$id}->{'port_list'}->[$ind1]);
+ # added for bnet JAM:
+ #push( @{$ports_href->{$id}->{'ip_list'}}, $tmp_href->{$id}->{'ip_list'}->[$ind1]);
+ #push( @{$ports_href->{$id}->{'calib_list'}}, $tmp_href->{$id}->{'calib_list'}->[$ind1]);
+ push( @{$ports_href->{$id}->{'addr_list'}}, $addr);
+ push( @{$ports_href->{$id}->{'bufsize_list'}}, $activeSources_href->{'bufsize_list'}->[$ind2]);
+
+ }
+ }
+ }
+
+ # print Dumper $ports_href;
+}
+
+sub readActiveSources()
+{
+ #
+ # Read TRB-Net addresses of active data sources
+ #
+
+ my ($file, $activeSources_href) = @_;
+
+ my $fh = new FileHandle("$file", "r");
+
+ &isFileDefined($fh, $file);
+
+ my $SPACE = "";
+
+ # this one contains list of deactivated data sources
+ my $log="/home/hadaq/trbsoft/hadesdaq/evtbuild/tmp/EB_NotActiveSources.txt";
+ system("truncate -s 0 $log ;");
+ while(<$fh>){
+
+ #- Remove all comments
+ $_ =~ s{ # Substitue...
+ \# # ...a literal octothorpe
+ [^\n]* # ...followed by any number of non-newlines
+ }
+ {$SPACE}gxms; # Raplace it with a single space
+
+ #- Skip line if it contains only whitespaces
+ next unless(/\S/);
+
+ my ($addr, $astat, $sys, $size) = split(" ", $_);
+
+ if($astat == 0){
+ system("echo $addr >> $log 2>&1;");
+ next;
+ }
+ push( @{$activeSources_href->{'addr_list'}}, $addr);
+ push( @{$activeSources_href->{'bufsize_list'}}, &getBufSize($size));
+ }
+
+ $fh->close;
+}
+
+sub getBufSize()
+{
+ my ($bufSize) = @_;
+
+ if(lc($bufSize) eq "low"){
+ return $temp_args_href->{'Main'}->{'BUF_SIZE_LOW'};
+ }
+ elsif(lc($bufSize) eq "mid"){
+ return $temp_args_href->{'Main'}->{'BUF_SIZE_MID'};
+ }
+ elsif(lc($bufSize) eq "high"){
+ return $temp_args_href->{'Main'}->{'BUF_SIZE_HIGH'};
+ }
+ else{
+ print "Cannot understand $bufSize from data_sources.db.\n";
+ exit(0);
+ }
+}
+
+sub getIP_hex2dec()
+{
+ my ($ip_hex) = @_;
+
+ my $ip_dec;
+
+ if( $ip_hex =~ /0x(\w{2})(\w{2})(\w{2})(\w{2})/ ){
+ $ip_dec = hex($1) . "." . hex($2) . "." . hex($3) . "." . hex($4);
+ }
+ else{
+ print "getIP_hex2dec(): cannot extract ip address because of diferent format! Exit.";
+ exit(0);
+ }
+
+ return $ip_dec;
+}
+
+sub getPort_hex2dec()
+{
+ my ($port_hex) = @_;
+
+ my $port_dec;
+
+ if( $port_hex =~ /0x(\w+)/ ){
+ $port_dec = hex($1);
+ }
+ else{
+ print "getPort_hex2dec(): cannot extract port number because of diferent format! Exit.";
+ exit(0);
+ }
+
+ return $port_dec;
+}
+
+sub getIP_hades2gsi()
+{
+ my ($ip_hades) = @_;
+ my $ip_gsi = "";
+ if (exists $IP_GSI_href->{$ip_hades}){
+ $ip_gsi = $IP_GSI_href->{$ip_hades};
+ }
+ else {
+ print "getIP_hades2gsi(): WARNING - unknown hades ip $ip_hades";
+ }
+ return $ip_gsi;
+}
+
+
+
+
+sub isFileDefined()
+{
+ my ($fh, $name) = @_;
+
+ if(!$fh) {
+ my $txt = "\nError! Could not open file \'$name\'. Exit.\n";
+ print STDERR $txt;
+ print $txt;
+ exit(128);
+ }
+
+ return 0;
+}
+
+sub writeArgs2file()
+{
+ my $fileName = $0;
+
+ #- Replace .pl with .sh
+ $fileName =~ s/\.pl/\.sh/;
+
+ my $fh = new FileHandle(">./$fileName");
+ if(!$fh) {
+ my $txt = "\nError! Could not open file \"$fileName\" for output. Exit.\n";
+ print STDERR $txt;
+ print $txt;
+ exit(128);
+ }
+
+ my $current_dir = cwd();
+ my $ptogName = $0;
+
+
+ #- Write to the file the script name itself
+ print $fh $0;
+
+ #- Write to the file the arguments
+ foreach my $arg (@arg_list){
+ print $fh " $arg";
+ }
+ print $fh ";\n";
+
+ ## do not start te files by default. user should actively restart the file JAM 29-06-2018
+ # here put command to start the correct file prefix in bnet builders:
+ #my $prefix=$EB_Args_href->{0}->{'PREFIX'};
+ #print $fh "/usr/bin/wget -a /tmp/EB_filestart.log -O /tmp/EB_fileres.txt \"http://lxhadeb03:8090/Master/BNET/StartRun/execute?prefix=$prefix&oninit=10\"\n";
+
+
+ $fh->close();
+
+ system("chmod 755 ./$fileName");
+}
+
####config
my @all_boards =();
my @mdc_boards =(0x8015,0x8016,
- ###OEPS
-# 0x2010,0x2011,0x2012,0x2013,0x2014,0x2015,0x2016,0x2017,0x2018,0x2019,0x201a,
-# 0x201b,0x201c,0x201d,0x2020,0x2021,0x2022,0x2023,0x2024,0x2025,0x2026,0x2027,
-# 0x2028,0x2029,0x202a,0x202b,0x202c,0x202d,0x2030,0x2031,0x2032,0x2033,0x2034,
-# 0x2035,0x2036,0x2037,0x2038,0x2039,0x203a,0x203b,0x203c,0x203d,0x2040,0x2041,
-# 0x2042,0x2043,0x2044,0x2045,0x2046,0x2047,0x2048,0x2049,0x204a,0x204b,0x204c,
-# 0x204d,0x2050,0x2051,0x2052,0x2053,0x2054,0x2055,0x2056,0x2057,0x2058,0x2059,
-# 0x205a,0x205b,0x205c,0x205d,0x2100,0x2101,0x2102,0x2103,0x2104,0x2105,0x2106,
-# 0x2107,0x2108,0x2109,0x210a,0x210b,0x210c,0x210d,0x210e,0x210f,0x2110,0x2111,
-# 0x2112,0x2113,0x2114,0x2115,0x2116,0x2117,0x2118,0x2119,0x211a,0x211b,0x211c,
-# 0x211d,0x211e,0x211f,0x2120,0x2121,0x2122,0x2123,0x2124,0x2125,0x2126,0x2127,
-# 0x2128,0x2129,0x212a,0x212b,0x212c,0x212d,0x212e,0x212f,0x2130,0x2131,0x2132,
-# 0x2133,0x2134,0x2135,0x2136,0x2137,0x2139,0x213a,0x213b,0x213c,0x213d,0x213e,
-# 0x213f,0x2140,0x2141,0x2142,0x2143,0x2144,0x2145,0x2146,0x2147,0x2148,0x2149,
-# 0x214a,0x214b,0x214c,0x214d,0x214e,0x214f,0x2150,0x2151,0x2152,0x2153,0x2154,
-# 0x2155,0x2156,0x2157,0x2158,0x2159,0x215a,0x215b,0x215c,0x215d,0x215e,0x215f,
-# 0x2200,0x2201,0x2202,0x2203,0x2204,0x2205,0x2206,0x2207,0x2208,0x2209,0x220a,
-# 0x220b,0x220c,0x220d,0x220e,0x220f,0x2220,0x2221,0x2222,0x2223,0x2224,0x2225,
-# 0x2226,0x2227,0x2228,0x2229,0x222a,0x222b,0x222c,0x222d,0x222e,0x222f,0x2240,
-# 0x2241,0x2242,0x2243,0x2244,0x2245,0x2246,0x2247,0x2248,0x2249,0x224a,0x224b,
-# 0x224c,0x224d,0x224e,0x224f,0x2250,0x2251,0x2252,0x2253,0x2254,0x2255,0x2256,
-# 0x2257,0x2258,0x2259,0x225a,0x225b,0x225c,0x225d,0x225e,0x225f,0x2300,0x2301,
-# 0x2302,0x2303,0x2304,0x2305,0x2306,0x2307,0x2308,0x2309,0x230a,0x230b,0x230c,
-# 0x230d,0x230e,0x230f,0x2320,0x2321,0x2322,0x2323,0x2324,0x2325,0x2326,0x2327,
-# 0x2328,0x2329,0x232a,0x232b,0x232c,0x232d,0x232e,0x232f,0x2340,0x2341,0x2342,
-# 0x2343,0x2344,0x2345,0x2346,0x2347,0x2348,0x2349,0x234a,0x234c,0x234d,0x234e,
-# 0x234f,0x2350,0x2351,0x2352,0x2353,0x2354,0x2355,0x2356,0x2357,0x2358,0x2359,
-# 0x235a,0x235b,0x235c,0x235d,0x235e,0x235f,
# ###MDC Concentrator
0x1000,0x1001,0x1002,0x1003,0x1004,0x1010,0x1011,0x1012,0x1013,0x1014,0x1020,
0x1021,0x1022,0x1023,0x1024,0x1030,0x1031,0x1032,0x1033,0x1034,0x1040,0x1041,
0x1103,0x1104,0x1120,0x1121,0x1122,0x1123,0x1124,0x1140,0x1131,0x1132,0x1133,
0x1134,0x1140,0x1141,0x1142,0x1143,0x1144,0x1150,0x1151,0x1152,0x1153,0x1154,
###
-# 0x8100,0x8101,0x8110,0x8111
);
foreach my $p (0..3) {
0x7070,0x7071,0x7080,0x7081,0x7090,0x7091,
);
- my @tof_boards =(0x4c00,0x4c10,0x4c20,0x4c30,0x4c31,0x4c40,0x4c50,0x8600,0x8601,0x4800,0x4801,0x4802,0x4803,0x4810,0x4811,0x4812,0x4813,0x4820,0x4821,0x4822,
- 0x4823,0x4830,0x4831,0x4832,0x4833,0x4840,0x4841,0x4842,0x4843,0x4850,0x4851,
- 0x4852,0x4853,0x8400,0x8401,0x8410,0x8411, 0x4400,0x4410,0x4420,0x8700,0x8701,);
-
- my @rpc_boards = (0x8013,
+#TOF -> is TOF, RPC,FW
+ my @tof_boards =(
+#0x86c0,0x5c00,0x5c01,0x5c02,0x5c03,
+#0x86c1,0x5c10,0x5c11,0x5c12,0x5c13,
+#0x86c2,0x5c20,0x5c21,0x5c22,0x5c23,
+ 0x86c3,0x5c30,0x5c31,0x5c32,0x5c33,
+#0x86c4,0x5c40,0x5c41,0x5c42,0x5c43,
+#0x86c5,0x5c50,0x5c51,0x5c52,0x5c53,
+#0x86c6,0x5c60,0x5c61,0x5c62,0x5c63,
+
+0x84c0,0x84c1,0x84c2,0x84c3,0x84c4,0x84c5,
+0x5800,0x5801,0x5802,0x5803,0x5804,0x5805,0x5806,0x5807,0x5808,
+0x5810,0x5811,0x5812,0x5813,0x5814,0x5815,0x5816,0x5817,0x5818,
+0x5820,0x5821,0x5822,0x5823,0x5824,0x5825,0x5826,0x5827,0x5828,
+0x5830,0x5831,0x5832,0x5833,0x5834,0x5835,0x5836,0x5837,0x5838,
+0x5840,0x5841,0x5842,0x5843,0x5844,0x5845,0x5846,0x5847,0x5848,
+0x5850,0x5851,0x5852,0x5853,0x5854,0x5855,0x5856,0x5857,0x5858,
+);
+
+#RPC -> is ECAL
+ my @rpc_boards = (
0x8a00,0x8a01,0x8a02,0x8a03,0x8a04,0x8a05,
#0x6000,0x6001,0x6002,0x6003,0x6004,0x6005,0x6006,
0x6010,0x6011,0x6012,0x6013,0x6014,0x6015,0x6016,
0x6020,0x6021,0x6022,0x6023,0x6024,0x6025,0x6026,
- #0x6030,0x6031,0x6032,0x6033,0x6034,0x6035,0x6036
+ 0x6030,0x6031,0x6032,0x6033,0x6034,0x6035,0x6036,
0x6040,0x6041,0x6042,0x6043,0x6044,0x6045,0x6046,
0x6050,0x6051,0x6052,0x6053,0x6054,0x6055,0x6056,
+
+#STT2
+ 0x8b10, 0x8b11, 0x8b12, 0x8b13, 0x8b14, 0x8b15,
+ 0x6440, 0x6441, 0x6442, 0x6443, 0x6444, 0x6445,
+ 0x6450, 0x6451, 0x6452, 0x6453, 0x6454, 0x6455,
+ 0x6460, 0x6461, 0x6462, 0x6463, 0x6464, 0x6465,
+ 0x6470, 0x6471, 0x6472, 0x6473, 0x6474, 0x6475,
+
);
my @other_boards =(
#cts
0x0002,0x0003,
#central hub
- 0x8000,0x8012,0x8800,
+ 0x8000,0x8011,0x8012,0x8013,0x8014,0x8800, 0x8810,
#start/veto
# 0x4000,
#Pion
# 0x8900,0x3800,0x3801,
# 0x8910,0x3810,0x3811,
#Start TRB3
- 0x8880,0x5000,0x5002,0x5003,
+# 0x8880,0x5000,0x5002,0x5003,
#Hodo TRB3
# 0x8890,0x5010,0x5011,0x5012,0x5013
- 0x0100,0x0110, #trigger etc.
+ 0x0100,0x0110 #trigger etc.
);
###strip the register value
my $num_rpc_mistake = (scalar @{$rpc_results[4]});
my @sorted_rpc_results = sort @{$rpc_results[2]};
- my $rpc_title = "ECal";
+ my $rpc_title = "ECal/STT/fRPC";
my $rpc_value = "OK ".(scalar @rpc_boards);
if ($num_rpc_missing > 0) {$rpc_value = "$num_rpc_missing missing";}
if ($num_rpc_mistake > 0) {$rpc_value = "Check Script";}
if ($num_rpc_mistake > 0) {$rpc_longtext .= " Endp @{$rpc_results[4]} not known";}
$qastate = QA::GetQAState('below',$num_rpc_missing,@QA::RpcEndpMissingLimits);
- Hmon::Speak('rpcmiss',"$num_rpc_missing ECal Frontends missing") if($qastate > 60);
+ Hmon::Speak('rpcmiss',"$num_rpc_missing Forward Frontends missing") if($qastate > 60);
QA::WriteQALog($flog,"endp","rpc",$waittime,$qastate,
$rpc_title,$rpc_value,$rpc_longtext);
if($qastate > 60) {
jspc29 / hadesdaq.git / commitdiff