#!/usr/bin/perl

use File::Copy;
use Cwd qw(abs_path);
use strict;

# Execute a VITESS command pipe on a computer grid.

# This script is meant as a working template to submit a
# grid batch job using n_nodes of the grid, calling n_instances
# of parallel pipe instances on nodes.
# The pipe commands are taken from a file generated by
# "Save Grid Command" within VITESS.
# gridrun works with the Sun Grid Engine (SGE), SLURM batch system,
# and ssh remote command execution.

# M. Fromme, HZB, Jan 2013

# Support for optimization added, Oct 2013
# SLURM batch jobs, Nov 2013

###############################################################
### configuration

# root of work directories
my $wroot = "$ENV{HOME}/.vitess";

# generic job file name, should have special chars to hinder confusion with normal files
my $job_fn = 'job_.dat';

# grid engine commands to submit, query, and delete a batch job
my @SGECom = ('qsub', 'qstat -j', 'qdel -f');

# slurm commands to submit, query, and delete a batch job
my @SlurmCom = ('sbatch', 'squeue', 'scancel');

# first line contents for SGE and ssh batch jobs
my $batchhead_sge =<<'EOS';
#!/bin/bash
#$ -S /bin/bash
#$ -cwd
#$ -l vf=4G
EOS

my $batchhead_ssh = "\#!/bin/bash\n";

$_ = `uname -sm`;
chomp;
tr/ /_/;
my $sysid = $_;
my $sysrep = "_$_";


### end configuration
###############################################################


# The input file named $filename is a text file with variable definitions of
# V the source module directory
# P the working directory
#  and command option intro strings for the
# G random number generator
# and the command pipe.

# Grid execution has to deal with
# - different file systems, $V and $P may not be present on the grid
# - parallel execution of pipes

# Because execution files may be invisible on grid nodes, we copy them to the
# work directory. We copy all files from $P to pipe sub directories.
# We assume n_instances ( >= 1 ) of work nodes in the grid.
# We want to start parallel pipes.
# Example:
# I'm user JOE, my working dir is /net/home/JOE/vitess on the Linux server dirac at HZB.
# After proper building an instrument with VITESS I found my simulation does take so long
# I'd like to speed that up using the grid engine.
# So first I create the grd file /net/home/JOE/vitess/test.grd
# using the Vitess "Save Grid Command" menu. Contents of this file are
#   V=/net/home/JOE/vitess/MODULES
#   P=/net/home/JOE/vitess/FILES/SAN_DIR
#   G=ran3
#   $V/source_Linux_x86_64 -S1 ...

# To run 4 pipes on the grid engine I call
# gridrun test.grd -p 4
# By default this will be one batch job to run 4 pipes on a single node.
# To use more nodes, I specify a list of compute nodes like
# gridrun test.grd -p 4 --nodes 'all.q@node28 all.q@node29'
# which will run two batch jobs with 2 pipes each.
# The script finds an unused job number <run-nr>, creates ~/.vitess/w<run-nr>
# and temporary work directories ~/.vitess/w<run-nr>/1 to ~/.vitess/w<run-nr>/4 .
# For optimization jobs we use ~/.vitess/o<run-nr> as we keep these directories,
# to be purged only on demand.

# Because /net/home/JOE/vitess/MODULES/ may be invisible for dirac compute nodes (it is invisible),
# all used *_Linux_x86_64 executables are copied to ~/.vitess/w<run-nr>, plus this
# script itself, and referenced files from /net/home/JOE/vitess/FILES/SAN_DIR
# plus test.grd are copied to the temporary work directories.

# The information necessary to handle <run-nr> is comprised in the file  ~/.vitess/w<run-nr>/job_.dat
# lines contents in Jinfo key=value lines, key may be
# state
# time        time at start
# smode       submit mode
# instances   n instances
# nodes
# pfiles      pipe files
# jobname     job name
# outfile     pipe output file name
# qnumbers    sge job numbers

sub usage {

  print <<'EOS';
usage:
gridrun [option] [filename]

gridrun may be used 
A  to start a VITESS simulation batch job
B  to inform about active jobs
C  to finalize jobs
D  for parallel execution of pipes under the control of an optimization program

A) start a job
gridrun [option] filename [batchopt batcharg]
filename             name of a file generated by Vitess "Save Grid Command"
batchopt batcharg    additional arguments for the SGE batch system like -l cpu=8
                     may be repeated
options may be
-p number            number of parallel pipes to start
-P number            generate a job for each parallel pipe, companion of --nodes=..
                     use either -p or -P
-m                   do not merge neutron trajectory output files
-r                   repeat random seed of source module to following modules
                     default: --Z values from grd file
--v=mode             verbose mode (0:default,silence  1:show some actions 2:DEBUG)
--nodes=node-list    list of names of SGE queues to be used
                     enclose lists if more than one queue given, like 'all.q@node28 all.q@node29'
--ssh=node-list      use ssh remote execution on these nodes
                     enclose lists like above, e.g. 'node28 node29'
--slnodes=number     use the SLURM batch system to start pipes on number nodes
--slpartition=name   name of a SLURM partition to be used
--bin=modules-dir    directory with VITESS binaries, default is to use those used in the grd file
                     or from the script environment as last resort
--files=file-list    needed input files, not listet as parameters in the pipe, but
                     necessary for execution, to be copied to the parameter direction
                     on execution nodes, like 'Isis1.water'
--param=dir          directory with parameter files, overrides name given in the grd file
--name=jobname       you may name the job
--opt=optstring      special bash command like 'ulimit -s 8192' or 'export LD_LIBRARY_PATH=...'
                     for generated  bash command files, may be repeated
--traj=n             explicitly give the number of neutron trajetories to simulated,
                     n overrides value in grd file
--T=number           override --T parameter in grid file to always use number helper threads

B) inform about jobs
gridrun option
option may be
-I                   show states of all known jobs
-i run-nr            show state of job run-nr

C) finalize jobs
gridrun option
option may be
-F                   finalize all completed, but still not merged, jobs
-f run-nr            finalize job run-nr
--merge=run-nr       finalize job run-nr, keep temporary directories
--abort=run-nr       aborts job 5 if still waiting for completion
--delete=run-nr      deletes all information about job 5, aborting it before, if still active

D) Optimization
-o filename          starts parallel pipes in an optimization step,
                     values given in this file, default for filename is Pcomm.dat
--sini=filename      name of ini file, default sim_param.ini
--sopt=filename      name of opt. param file, default opt_param.ini
--instr=filename     name of instrument file, default std_instr.cmd

EOS
  exit 0;
}

my ($filename, $n_nodes, $n_instances, $vitess_binaries, $vitess_param, $extra_files,
    $grid_mod_dir, $grid_work_dir, $submit_mode, $ssh_shell,
    $verbose, $force_delete, $do_not_delete,
    $op_mode, $jobname, $neutron_filename, $neutrons, $trajectories, $threads,
    $slnodes, $slpartition, $slpernode,
    $run_nr, $abort_run, $delete_run, $repeatseed,
    $merge_again, $ppjob,
    $pipe_str,
    $opt_dat, $opt_rdir, $opt_pdir, @FileOfMerit, @SimPars, @SimDat,
    @Node,  @Pipe, @VBinary, @VFile, @BashOption,
    %Batcharg);

my $merge_neutron_trajectories = 1;

my $dir_pattern = 'w';          # working diretory pattern, may be w for normal grid jobs
                                # or o for optimization jobs

my $gsl_rng_type = 'ran3';      # default PRNG

my  $sini_file = 'sim_param.ini';
my  $sopt_file = 'opt_param.ini';
my  $instr_file = 'std_instr.cmd';
my  $opt_dat_default = 'Pcomm.dat';

my $start_time = time;

# parse script arguments

while ($_ = shift) {
  if (/^-(.+)/) {
    $_ = $1;
    if ($_ eq 's') {
      # special mode, not interactive
      $op_mode = 'submit';
      last;
    }
    if (/^-([a-zA-Z]+)=([^=].*)/) {
      # -- options
      my $opt = lc($1);
      &usage if $_ eq 'help' || $_ eq 'h';
      # delete string quotes
      $_ = $2;
      s/^['"]//;
      s/['"]$//;
      my $arg = $_;
      if ($opt =~ /^node/ || $opt eq 'ssh') {
        $submit_mode = $opt eq 'ssh' ? 'ssh' : 'sge';
        # silently discard comma separators
        tr/,/ /;
        @Node = split;
        myexit("please give a list of $opt nodes") unless @Node;
      } elsif ($opt =~ /^slno/) {
        $submit_mode = 'slurm';
        $slnodes = $arg;
      } elsif ($opt =~ /^slpart/) {
        $slpartition = $arg;
      } elsif ($opt eq 'bin') {
        $vitess_binaries = $arg;
      } elsif ($opt =~ /^par/) {
        $vitess_param = $arg;
      } elsif ($opt =~ /files/) {
        myexit('extra files only relative to parameter directory')
          if $arg =~ /[\$\/]/;
        $extra_files = $arg;
      } elsif ($opt eq 'abort') {
        $op_mode = 'finalize';
        $abort_run = $arg;
      } elsif ($opt =~ /^del/) {
        $op_mode = 'finalize';
        $force_delete = 1;
        $delete_run = $arg;
      } elsif ($opt eq 'name') {
        $jobname = $arg;
      } elsif ($opt =~ /merge/) {
        $op_mode = 'finalize';
        $do_not_delete = 1;
        $run_nr = $arg;
        $merge_again = 1 if $opt =~ /again/;
      } elsif ($opt =~ /^traj/) {
        $trajectories = $arg;
      } elsif ($opt eq 'sini') {
        $sini_file = $arg;
      } elsif ($opt eq 'sopt') {
        $sopt_file = $arg;
      } elsif ($opt eq 'instr') {
        $instr_file = $arg;
      } elsif ($opt eq 't') {
        $threads = $arg;
      } elsif ($opt eq 'v') {
        $verbose = $arg;
      } elsif ($opt =~ /^opt/) {
        push @BashOption, $arg;
      } else {
        &usage;
      }
    } else {
      # - options
      &usage if $_ eq 'help' || $_ eq 'h';
      if ($_ eq 'm') {
        $merge_neutron_trajectories = 0;
      } elsif ($_ eq 'i') {
        $op_mode = 'info';
        $run_nr = shift;
      } elsif ($_ eq 'I') {
        $op_mode = 'info';
      } elsif ($_ eq 'f') {
        $op_mode = 'finalize';
        $force_delete = 1;
        $run_nr = shift;
      } elsif ($_ eq 'F') {
        $force_delete = 1;
        $op_mode = 'finalize';
      } elsif ($_ eq 'o') {
        $op_mode = 'optimization';
      } elsif ($_ eq 'p') {
        $n_instances = shift;
      } elsif ($_ eq 'P') {
        $n_instances = shift;
        $ppjob = 1;
      } elsif ($_ eq 'r') {
        $repeatseed = 1;
      }
    }
  } elsif ($filename) {
    $Batcharg{$_} = shift;
  } else {
    $filename = $_;
  }
}

if ($op_mode eq 'optimization') {
  $opt_dat = $filename ? $filename : $opt_dat_default;
  &doOptimization;
} elsif ($op_mode eq 'submit') {
  startPipes(@ARGV);
} elsif ($op_mode eq 'info') {
  &infoJobs;
} elsif ($op_mode eq 'finalize') {
  &finalizeJobs;
} elsif ($filename eq '') {
  &infoJobs;
} else {
  &readGrdFile;
  &promptMissingParameters;
  &createWorkdir;
  &startExecution;
}


sub infoJobs {
  my $anyaction;

  if (-d $wroot) {

    if ($run_nr) {
      $grid_work_dir = "$wroot/$dir_pattern$run_nr";
      if (-d $grid_work_dir) {
        infoJob($run_nr, $grid_work_dir);
        $anyaction = 1;
      } else {
        print "no info for run number $run_nr.\n";
      }
    } else {
      my @Runs;
      opendir D, $wroot;
      while ($_ = readdir D) {
        if (/^$dir_pattern([0-9]+)$/) {
          push @Runs, $1 if -d "$wroot/$_";
        }
      }
      closedir D;
      foreach (sort @Runs) {
        infoJob($_, "$wroot/$dir_pattern$_");
      }
      $anyaction = 1 if @Runs;
    }
  }
  return if $anyaction;
  $_ = abs_path($0);
  print "No VITESS jobs known. Call\n$_ -h\nfor help.\n";
}


sub infoJob {
  my ($j, $jdir) = @_;

  my ($newstate, $i, $pipe_not_started, %Jinfo);
  my $jobfile = "$jdir/$job_fn";

  # read job state file
  open F, $jobfile;
  while (<F>) {
    next unless /^([a-z0-9]+)=(.+)/;
    $Jinfo{$1} = $2;
  }
  close F;

  $_ = $Jinfo{jobname};
  my $jn = $_ ? " ($_)" : '';

  my $oldstate = $newstate = $_ = $Jinfo{state};
  if ($_ eq 'finalized') {
    print "Job $j$jn has already been finalized to $jdir .\n";
    return;
  }

  my $pipedone=1;

  # if it's still running, query the state
  if ($_ eq 'running') {
    for ($i=1; -d "$jdir/$i"; $i++) {
      # if a file PIPEDONE_ exists, this pipe has finished
      next if -f "$jdir/$i/PIPEDONE_";
      $pipedone = 0;
      $pipe_not_started = 1 unless -f "$jdir/$i/PIPEPID_";
    }
    if ($pipedone) {
      # all pipes have finished
      $newstate = 'finished';
    }
  }

  # show state

  if ($newstate eq 'running') {
    print "Job $j$jn is running.";
    print " Some pipes did not start yet." if $pipe_not_started;
    print "\n";
  } elsif ($newstate eq 'finished') {
    print "Job $j$jn pipes have finished, you may finalize it.\n";
  }

  # if state had changed, update the state file
  return if $newstate eq $oldstate;
  $Jinfo{state} = $newstate;
  open OF, ">$jobfile";
  print OF "$_=$Jinfo{$_}\n" foreach keys %Jinfo;
  close OF;
}


sub finalizeJobs {
  return unless (-d $wroot);
  if ($delete_run) {
    $_ = $delete_run;
  } elsif ($abort_run) {
    $_ = $abort_run;
  } else {
    $_ = $run_nr;
  }
  if ($_) {
    $grid_work_dir = "$wroot/$dir_pattern$_";
    if (-d $grid_work_dir) {
      finalizeJob($_, $grid_work_dir, $delete_run ? 2 : ($abort_run ? 1 : 0));
    } else {
      print "Job $_ does not exist.\n";
    }
  } else {
    my @Runs;
    opendir D, $wroot;
    while ($_ = readdir D) {
      if (/^$dir_pattern([0-9]+)$/) {
        push @Runs, $1 if -d "$wroot/$_";
      }
    }
    closedir D;
    foreach (sort @Runs) {
      finalizeJob($_, "$wroot/$dir_pattern$_", 0);
    }
  }
}

sub finalizeJob {
  my ($j, $jdir, $fmode) = @_;

  DeBug("finalizeJob $j, $jdir, $fmode", 1);

  # mode may be 0:normal  1:abort  2:delete

  # read job state file
  my ($i, $newstate, %Jinfo);
  my $jobfile = "$jdir/$job_fn";
  unless (-s $jobfile) {
    DeBug("can't read job file $jobfile", 1);
    deleteJob($j) if $fmode == 2;
    return;
  }

  open F, $jobfile;
  while (<F>) {
    next unless /^([a-z0-9]+)=(.+)/;
    $Jinfo{$1} = $2;
  }
  close F;

  my $jn = "Job $j";
  $_ = $Jinfo{jobname};
  $jn .= " ($_)" if $_;

  my $oldstate = $newstate = $_ = $Jinfo{state};
  if ($_ eq '') {
    if ($fmode == 2) {
      deleteJob($j);
    } else {
      print "$jn is krank!\n";
    }
    return;
  }

  # if it's already finalized, show that information and return
  if ($_ eq 'finalized' && !$merge_again) {
    if ($fmode == 2) {
      deleteJob($j);
    } else {
      print "$jn has been finalized.\n";
    }
    return;
  }

  # if it's in state running, query the state
  if ($_ eq 'running') {
    my $s;
    for ($i=1; -d "$jdir/$i"; $i++) {
      $s .= "pipes of $jn have not finished yet.\n"
        unless -f "$jdir/$i/PIPEDONE_";
    }
    if ($fmode > 0) {
      abortJob($j, %Jinfo) if $s;
      if ($fmode == 2) {
        deleteJob($j);
        return;
      }
    } elsif ($s) {
      print $s;
    }
  } elsif ($fmode == 2) {
    deleteJob($j);
    return;
  }

  # job has finished, merge results
  my $rc = mergeResults($jdir, $Jinfo{time}, $Jinfo{pfiles}, $Jinfo{outfile});

  if ($rc) {
    DeBug("mergeResults returns :$rc:", 1);
    print "Some results could not be merged, keeping file state as finished.\n";
    return;
  }

  DeBug("set state finalized", 1);
  # update the state file
  $Jinfo{state} = 'finalized';
  open OF, ">$jobfile";
  print OF "$_=$Jinfo{$_}\n" foreach keys %Jinfo;
  close OF;

  deleteTempWorkDirs($jdir) unless $do_not_delete;

  # show final state
  print "$jn has been finalized.\nPlease find the results in directory $jdir .\n";
}


sub abortJob {

  my $j = shift;
  my %Jinfo = @_;
  my $jdir = "$wroot/$dir_pattern$j";

  DeBug("abortJob $j, $jdir", 1);

  my ($i, $pid, $pfn, $host, $jnr, $anyaction, @Jobdelist);

  if ($Jinfo{smode} eq 'ssh') {

    for ($i = 1; -d "$jdir/$i"; $i++) {
      next if -f "$jdir/$i/PIPEDONE_";
      $pfn = "$jdir/$i/PIPEPID_";
      if (-s $pfn) {
        open F, $pfn;
        $_ = <F>;
        close F;
        if (/^([0-9+]+) (.+)/) {
          $pid = $1;
          $host = $2;
          # issue kill command via ssh
          $_ = `ssh $host kill $pid`;
          $anyaction = 1;
        }
      }
    }

  } elsif ($Jinfo{smode} eq 'sge') {

    $_ = $Jinfo{qnumbers};
    my @Jobs = split;
    foreach $jnr (@Jobs) {
      $_ = `$SGECom[1] $jnr 2>&1`;
      next if /Following jobs do not exist/g;
      push @Jobdelist, $jnr;
    }
    DeBug("abortlist :@Jobdelist:", 1);
    if (@Jobdelist) {
      # try to abort remaining jobs with qdel -f
      $_ = `$SGECom[2] @Jobdelist 2> /dev/null`;
      $anyaction = 1;
    }

  } elsif ($Jinfo{smode} eq 'slurm') {

    $_ = `$SlurmCom[2] $Jinfo{qnumbers} 2> /dev/null`;
    $anyaction = 1;

  } else {
    return;
  }

  return unless $anyaction;

  # change jobfile to indicate abortion
  $Jinfo{state} = 'aborted';
  open OF, ">$jdir/$job_fn";
  print OF "$_=$Jinfo{$_}\n" foreach keys %Jinfo;
  close OF;
  # wait some time to let SGE finish
  sleep 2 if @Jobdelist;
}


sub deleteJob {
  my $d = "$wroot/$dir_pattern$_[0]";
  DeBug("deleteJob $_[0], $d", 1);
  deleteTempWorkDirs($d);
  delDir($d);
}

sub deleteTempWorkDirs {
  my ($jdir, $delall) = @_;

  return unless -d $jdir;

  # delete pipe instance sub directories
  my $i = 1;
  $i++ while delDir("$jdir/$i");

  # recursively delete FILES sub directory
  $_ = `rm -r $jdir/FILES` if -d "$jdir/FILES";

  # get rid of pipe std dummies, the copy of gridrun, and VITESS binaries
  opendir D, $jdir;
  while ($_ = readdir D) {
    next if /^\./;
    my $fn = "$jdir/$_";
    unlink $fn if $delall || /^[0-9]+\.(log|stderr)\.txt$/ || $_ eq 'gridrun_' ||
      (/_$sysid$/ && -x $fn);
  }
  closedir D;
}

sub delDir {
  my $d = $_[0];
  return 0 unless -d $d;
  opendir D, $d;
  while ($_ = readdir D) {
    unlink "$d/$_" unless /^\./;
  }
  closedir D;
  rmdir $d;
  1;
}


sub startExecution {

  my ($i, $fn, $tdir, $nodes, $cs, @PFiles, @Match);

  if ($slnodes ne '') {
    $nodes = $slnodes;
    # put equal load to nodes
    $slpernode = int($n_instances / $nodes);
    # cut off surplus instances
    $n_instances = $nodes * $slpernode;
  } else {
    $nodes = 0 + @Node;
  }

  # gather existing parameter files
  foreach (@VFile) {
    $fn = "$vitess_param/$_";
    push @PFiles, $_ if -f $fn && -s _;
  }

  # create temp. directories for pipe/optimization instances
  for ($i=1; $i<=$n_instances; $i++) {
    $tdir = "$grid_work_dir/$i";
    mkdir $tdir, 0700;
    myexit ("unable to create temp. directory $tdir") unless -d $tdir;
    # copy parameter files to temp. directory
    copy("$vitess_param/$_", "$tdir/$_") foreach @PFiles;
  }

  DeBug('create command files');
  &createCommandfiles;

  DeBug('copy binaries');
  # copy VITESS binaries to work dir
  foreach (@VBinary) {
    copy("$vitess_binaries/$_", "$grid_work_dir/$_");
    chmod 0700, "$grid_work_dir/$_";
  }

  # copy part of the FILES subtree to the work directory
  my $vf = abs_path("$vitess_binaries/../FILES");
  my $fdir = "$grid_work_dir/FILES";
  mkdir $fdir, 0700;
  # copy some .dat files
  foreach (qw(ErrorTable)) {
    copy("$vf/$_.dat", "$fdir/$_.dat");
  }
  # EssModChar.dat quirks
  $_ = "$fdir/moderators";
  mkdir $_, 0700;
  $_ .= '/ESS';
  mkdir $_, 0700;
  copy("$vf/moderators/ESS/EssModChar.dat", "$_/EssModChar.dat");

  if ($submit_mode ne 'slurm') {

    # copy this script to work directory
    $cs = "$grid_work_dir/gridrun_";
    copy($0, $cs);
    chmod 0700, $cs;

    # match pipe instances to nodes
    @Match = matchPipesToNodes($nodes) if $submit_mode ne '';

  }

  # write state file
  my $jobinfo =<<EOS;
state=running
time=$start_time
smode=$submit_mode
instances=$n_instances
nodes=@Node
pfiles=@VFile
jobname=$jobname
outfile=$neutron_filename
EOS

  open OF, ">$grid_work_dir/$job_fn";
  print OF $jobinfo;
  close OF;

  # This script has been copied to the work directory, and will be called remotely
  # to fire the VITESS pipes.

  if ($submit_mode eq '') {

    # execute pipes locally
    system "$cs -s $run_nr 1 $n_instances &>$grid_work_dir/1.log.txt";

  } elsif ($submit_mode eq 'sge') {

    my @SGEJobNr;
    my $jfn = "$grid_work_dir/$job_fn";
    open OF, ">$jfn";
    print OF $jobinfo, "qnumbers=\n";
    close OF;
    foreach (@Match) {
      my ($node, $first, $last) = split;
      # create a job file for SGE
      my $fn = "$grid_work_dir/job_$first";
      open OF, ">$fn";
      print OF $batchhead_sge;
      print OF "\#\$ $_ $Batcharg{$_}\n" foreach keys %Batcharg;
      print OF "$cs -s $run_nr $first $last\n";
      close OF;
      chmod 0700, $fn;
      # extract a resonable sge job name
      $_ = "$jobname-$node-$first";
      tr/.a-zA-Z0-9_-/_/c;
      $_= "v$_" unless /^[a-zA-Z]/; # sge job name must begin with alpha char
      $_ = "$SGECom[0] -N $_";
      # submit that job
      # we do a cd first to have sge log files there
      $_ = `cd $grid_work_dir ; $_ -q $node $fn`;
      push @SGEJobNr, $1 if /Your job ([0-9]+) .+has been submitted/;
    }
    # write state file again, now with job numbers appended
    open OF, ">$jfn";
    print OF $jobinfo, "qnumbers=@SGEJobNr\n";
    close OF;

  } elsif ($submit_mode eq 'slurm') {

    # write shell file w.sh which will connect job i to work sub directory i+1
    my $fn = "$grid_work_dir/w.sh";
    open OF, ">$fn";
    $_ = '`expr $SLURM_PROCID + 1`';
    print OF <<EOS;
#!/bin/bash
$grid_work_dir/$_/command_
EOS
    close OF;
    # set executable
    chmod 0700, $fn;

    # write slurm batch command file
    my $jobsh = "$grid_work_dir/job_.sh";
    open OF, ">$jobsh";
    print OF <<'EOS';
#!/bin/bash
#SBATCH --exclusive
#SBATCH --error=job.%J.err
#SBATCH --output=job.%J.out
EOS
    print OF <<EOS;
#SBATCH --job-name=$jobname
#SBATCH --nodes=$slnodes
#SBATCH --ntasks-per-node=$slpernode
EOS
    print OF "#SBATCH --partition=$slpartition\n" if $slpartition ne '';
    print OF "srun -v -O w.sh\nwait\n";
    close OF;
    chmod 0700, $jobsh;

    # submit batch file
    # cd first to have log files in work directory
    DeBug("cd $grid_work_dir ; $SlurmCom[0] $jobsh", 1);
    $_ = `cd $grid_work_dir ; $SlurmCom[0] $jobsh`;
    my $answer = $_;
    $_ = /Submitted batch job ([0-9]+)/ ? $1 : '';
    DeBug($answer, 1);

    # attach job number to job dat file
    open OF, ">>$grid_work_dir/$job_fn";
    print OF "qnumbers=$_\n";
    close OF;

  } else {

    # We start remote commands with ssh.
    # Those commands should work in background at the remote side, and for that
    # we need to redirect stdout + stderr output of the remote command to files,
    # as ssh would wait for stdout + stderr completion otherwise.
    # Unfortunately redirection on the remote side depends on the shell executed there,
    # and we may not force a special shell.

    my ($bc, $logfn);
    foreach (@Match) {
      my ($node, $first, $last) = split;
      my $c = "$cs -s $run_nr $first $last";
      $logfn = "$grid_work_dir/$first.log.txt";
      DeBug("ssh shell is $ssh_shell");
      if ($ssh_shell eq 'tcsh') {
        $bc = "($c > $logfn) >& $grid_work_dir/$first.stderr.txt";
      } else {
        $bc = "$c &>$logfn";
      }
      DeBug("ssh $node '$bc &'");
      system "ssh $node '$bc &'";
    }
  }

  print "Job $run_nr started.\n" unless $opt_dat;
}


sub matchPipesToNodes {

  my $nodes = $_[0];

  DeBug("match $nodes nodes to $n_instances instances", 1);

  # match pipe instances to execution nodes / queues

  return ("$Node[0] 1 $n_instances") if ($nodes <= 1 && !$ppjob) || $n_instances <= 1;

  my ($firsti, $nexti, $i, @I, @Match);
  my $node = 0;

  if ($n_instances <= $nodes) {
    # put a pipe on the first nodes each
    for ($i=1; $i <= $n_instances; $i++) {
      push @Match, "$Node[$node] $i $i";
      $node++;
    }
  } elsif ($submit_mode eq 'sge'  && $ppjob) {
    # each pipe is done per own sge job
    for ($i=1; $i <= $n_instances; $i++) {
      $_ = $node++ % $nodes;
      push @Match, "$Node[$_] $i $i";
    }
  } else {
    my $inst_per_node = int($n_instances / $nodes);
    # put inst_per_node pipes on execution nodes / queues 0..nodes-1
    # the last node takes the rest
    # the nodes 0..nodes-1 perform inst_per_node pipes
    $nexti = 1;
    for ($i=1; $i < $nodes; $i++) {
      $firsti = $nexti;
      $nexti = $firsti + $inst_per_node;
      $_ = $nexti - 1;
      push @Match, "$Node[$node] $firsti $_";
      $node++;
    }
    # the last node takes the rest
    push @Match, "$Node[$node] $nexti $n_instances";
  }

  @Match;
}

sub createWorkdir {
  mkdir $wroot, 0700 unless -d $wroot;
  myexit("unable to create directory $wroot") unless -d $wroot;

  # find a free run number
  if ($dir_pattern ne 'w') {
      my $now = time;
      for ($run_nr = 1; ; $run_nr++) {
	  $grid_work_dir = "$wroot/$dir_pattern$run_nr";
	  last unless -d $grid_work_dir;
	  $_ = "$grid_work_dir/FINISHED_";
	  next unless -f $_;  # old job not fininished properly
	  next if $now - (stat($_))[9] < 300; # re-use only after 5 minutes
	  deleteTempWorkDirs($grid_work_dir, 1);
	  last;
      }
  } else {
      $run_nr = 1;
      opendir D, $wroot;
      while ($_ = readdir D) {
	  if (/^$dir_pattern([0-9]+)$/) {
	      $_ = $1;
	      $run_nr = $_ + 1 if $_ >= $run_nr;
	  }
      }
      closedir D;
      $grid_work_dir = "$wroot/$dir_pattern$run_nr";
  }
  mkdir $grid_work_dir, 0700 unless -d $grid_work_dir;
  myexit("unable to create directory $grid_work_dir") unless -d $grid_work_dir;
}


sub readGrdFile {
  # read a VITESS grid file
  # main contents are the pipe command and shell variable definition lines

  myexit("can't read grid job file $filename") unless -s $filename && -r _;

  my ($fmoddir, $fpdir);
  open F, $filename;
  <F>;                          # skip bang line
  while (<F>) {
    chomp;
    next if /^\s*\#/;           # skip comments
    if (/^([VPG])=(.+)$/) {
      # remember special comments
      if ($1 eq 'V') {
        $fmoddir = $2;
      } elsif ($1 eq 'P') {
        $fpdir = $2;
      } else {
        $gsl_rng_type = $_ = lc($2);
        myexit('unknown GSL PRNG specified') unless /^(ran3|taus|gfsr4|mt19937|ranlux)$/;
      }
    } elsif (/\|/) {
      $pipe_str = $_;
      # remove blanks before and after | chars
      s/ +\|/\|/g;
      s/\| +/\|/g;
      @Pipe = split('\|', $_);
      last;
    }
  }
  close F;
  myexit("found no VITESS pipe in $filename") unless @Pipe;

  analyzePipe($fmoddir, $fpdir);
}

sub analyzePipe {

  my ($fmoddir, $fpdir) = @_;
  # look for binaries and parameter file names accessed in the pipe
  my ($fn, $c, @C, @NewPipe, %V, %F);
  my $firstmod = 1;
  my $i = 1;

  my $t = ' --L$T/g';   # is replacement string for logfile parameters
  foreach $c (@Pipe) {
    $_ = $c;
    # force --T$threads
    s/--T[0-9]+/--T$threads/ if $threads;

    @C = split;
    # correct any log filename spec to be $T/g<i>
    s/ +--L[^ ]+/$t$i/;
    $i++;
    push @NewPipe, $_;
    $_ = shift @C;

    $V{$1} = 1 if /^\$V\/(.+)$/;
    if ($firstmod && ! /^\$V\/source/) {
      if ($c =~ m@--f([a-zA-Z0-9/._-]+)@) {
        my $nfile = $1;
        myexit("can not read input neutron trajectory file $nfile") unless -s $nfile && -r _;
      } else {
        myexit('first module should be a source module, or input neutron trajectory file should be given by absolute path');
      }
    }

    foreach (@C) {
      next unless /^-([a-zA-Z])(.+)/;
      my $optchar = $1;
      my $arg = $_ = $2;
      if (/^\$P\/(.+)/) {
        # true file in parameter directory
        $F{$1} = 1;
      } else {
        if ($firstmod) {
          if ($optchar eq 'n') {
            $neutrons = $arg;
            next;
          }
        }
        myexit('all parameter files in the pipe command should be contained in the parameter directory') if m@/@;
        # skip number (int or float) value parameters
        next if /^[0-9.-]+$/;
        next if /^[0-9.-]+e[0-9-]+$/;
        # what remains could be file names, relative to the parameter directory path
        $F{$_} = 1;
      }
    }
    $firstmod = 0;
  }

  # add extra files from --file option
  if ($_ = $extra_files) {
    $F{$_} = 1 foreach split;
  }

  @Pipe = @NewPipe;
  @VFile = keys %F;

  # Extract binary names from pipe, and replace correct image names if necessary
  @VBinary = keys %V;
  my $rep;
  $_ = $VBinary[0];
  unless (/$sysid$/) {
    # The grd file has been exported under a different OS
    # We need to replace binary names
    if (/\.exe$/) {
      $rep = '.exe';
    } elsif (/(_Linux.*)$/) {
      $rep = $1;
    } elsif (/(_Darwin.*)$/) {
      $rep = $1;
    }
    @VBinary = ();
  }
  # reconstruct the pipe string, as we have to replace
  # - the number of neutrons of the source module
  # - the PRNG seed of the source module
  # - and may be binaray names

  my @OPipe = @Pipe;            # remember the pipe
  @Pipe = ();                   # to construct a new array
  my $firstmodule = 1;
  my (%VB);

  while ($_ = shift @OPipe) {
    unless ($opt_dat) {
      # reduced number of neutrons, as we have parallel pipes
      s/ -n$neutrons/ NNNNN/ if $firstmodule;
      # individual PRNG seed, to be filled in later
      s/ --Z[^ ]+ / ZZZZZ / if $firstmodule || $repeatseed;
    }
    $firstmodule = 0;
    if ($rep) {
      @C = split;
      # correct sysid in a shift/replace/unshift sequence
      $_ = shift @C;
      s/$rep$/$sysrep/;
      unshift @C, $_;
      push @Pipe, "@C";
      # remember the name of the binary
      /^\$V\/(.+)$/;
      $VB{$1} = 1;
    } else {
      push @Pipe, $_;
    }
  }
  @VBinary = keys %VB if $rep;

  $pipe_str = join(' | ', @Pipe);

  if ($merge_neutron_trajectories) {
    # has the last module some output file?
    $_ = $Pipe[$#Pipe];
    if (/ --F([a-zA-Z0-9_.-]+)/) {
      $_ = $1;
      $neutron_filename = $_ if $_ ne 'no_file';
    } elsif (/ --F\$P\/([a-zA-Z0-9_.-]+)/) {
      $neutron_filename = $1;
    }
  }
  $vitess_binaries = $fmoddir unless $vitess_binaries;
  $vitess_param = $fpdir unless -d $vitess_param;

}

sub checkSSH {
  if ($submit_mode eq 'ssh') {

    # We need pre-shared ssh keys to work without password input
    $_ = "$ENV{HOME}/.ssh";
    unless ((-r "$_/identity" && -r "$_/identity.pub") ||
            (-r "$_/id_dsa" && -r "$_/id_dsa.pub") ||
            (-r "$_/id_rsa" && -r "$_/id_rsa.pub")) {
      myexit('You need to have a ssh key pair generated by ssh-keygen, see man ssh');
    }
    while ($#Node < 0) {
      $_ = prompt('list of nodes to be used');
      tr/,/ /;
      @Node = split;
    }
    # We need to know about the remote shell at these nodes, and test a ssh connection.
    $_ = `ssh $Node[0] echo \$SHELL`;
    $ssh_shell = /tcsh/ ? 'tcsh' : 'sh';

  } elsif ($submit_mode eq 'sge') {

    while ($#Node < 0) {
      last if $n_instances == 1; # use the default queue
      $_ = prompt('list of SGE queues to be used');
      tr/,/ /;
      @Node = split;
    }
  }
}

sub promptMissingParameters {

  # check if binaries and parameter directory are accessible here.

  my $binfound = -d $vitess_binaries;
  unless ($binfound) {
    # if not stated explicitly, we try to find them in a MODULES directory
    # in the environement of the gridrun script.
    $_ = abs_path($filename);
    m@^(.+)/[^/]+@;
    $_ = $1;
    $_ = <$_/*/$VBinary[0]>;
    if ($_) {
      m@^(.+)/[^/]+$@;
      $vitess_binaries = $1;
      $binfound = -d $vitess_binaries;
    } else {
      $binfound = 0;
    }
  }
  while (! $binfound) {
    if ($vitess_binaries) {
      $binfound = 1;
      foreach my $fn (@VBinary) {
        $_ = "$vitess_binaries/$fn";
        unless (-x $_) {
          print "can't use VITESS binary $_!\n";
          $binfound = 0;
          last;
        }
      }
      last if $binfound;
    }
    $vitess_binaries = prompt('directory with VITESS binaries');
  }

  while (! -d $vitess_param) {
    print "can't read parameter directory $vitess_param\n" if $vitess_param;
    $vitess_param = prompt('parameter directory');
  }

  while (! $n_instances =~ /^[1-9][0-9]*$/) {
    $n_instances = prompt('number of parallel pipe instances');
  }

  while ($submit_mode ne 'sge' && $submit_mode ne 'ssh' && $submit_mode ne 'slurm') {
    $submit_mode = prompt('method to submit jobs, either sge, ssh, or slurm');
  }

  &checkSSH if $submit_mode =~ /sge|ssh/;

  while ($jobname eq '') {
    $jobname = prompt('job name');
  }

}


sub prompt {
  my ($a, $b) = @_;
  $b = ':' unless $b;
  print substr("$a                                               ",0,48), "$b ";
  $_ = <STDIN>;
  /(.+)/;
  $1;
}


sub createCommandfiles {

  # Create command files for each pipe instance.
  # These are augmented grd files, with specific work directories and PRNG seeds.

  my ($i, $j, $cs, $tdir, $tstr, $lasti, $pn, @TFile);

  # reduced number of neutrons, as we have parallel pipes (except optimization runs)
  $pn = int(($trajectories ? $trajectories : $neutrons) / $n_instances) unless $opt_dat;

  # @TFile is array of all temp. file names
  $lasti = 0 + @Pipe;
  for ($i=1; $i <= $lasti; $i++) {
    push @TFile, "\$T/g$i";
  }
  $tstr = "@TFile";

  for ($i=1; $i<=$n_instances; $i++) {
    $tdir = "$grid_work_dir/$i";
    $cs = "$tdir/command_";
    open OF, ">$cs";
    print OF "\#!/bin/bash\n";
    print OF "$_\n" foreach @BashOption;
    $_ = $pipe_str;
    if ($opt_dat) {
      # replace values for this opt. run
      my @V  = split ' ', $SimDat[$i - 1];
      for ($j=0; $j < 0+@SimPars; $j++) {
        s/VVV${j}VVV/$V[$j]/;
      }
    } else {
      # replace random seed of source module
      s/ZZZZZ/--Z$i/g;
      # insert reduced number of neutrons
      s/NNNNN/-n$pn/;
    }

    # ulimit explitly, because on dirac thread creation fails if unlimited

    print OF <<EOS;
cd $tdir
ulimit -Ss 8192
V=$grid_work_dir
P=$tdir
export VITESSROOT=$grid_work_dir/
export GSL_RNG_TYPE=$gsl_rng_type
T=`mktemp -d /tmp/v_$ENV{USER}_XXXXXX` || exit 1
echo "\$\$ \$HOSTNAME" > PIPEPID_
$_
cat -s $tstr > gridlog 2> /dev/null
rm \$T/*
rmdir \$T
touch PIPEDONE_
EOS

    close OF;
    # set executable
    chmod 0700, $cs;
  }
}


sub startPipes {
  my ($run_nr, $first, $last) = @_;

  # Start pipe instances by spawning the specific command_ scripts.
  # This routine is called when submitted with SGE or ssh, and starts
  # all pipes dedicated to a compute node.

  exit 2 unless $run_nr >= 1 && $first ne '' && $last ne '' && $first >= 1 && $first <= $last;

  $grid_work_dir = "$wroot/$dir_pattern$run_nr";
  exit 2 unless -d $grid_work_dir;

  my ($i, $tdir);
  # check if temp. directories exist
  for ($i=$first; $i<=$last; $i++) {
    exit 2 unless -d "$grid_work_dir/$i";
    # do not restart by accident
    exit 2 if -f "$grid_work_dir/$i/PIPEPID_";
  }

  # If we are in ssh mode, all shell files are started in background,
  # for SGE the last shell works in foreground, so that the job remains active.
  my $is_ssh = 1;
  open F, "$grid_work_dir/job_.dat";
  while (<F>) {
    if (/^qnumbers=/) {
      $is_ssh = 0;
      last;
    }
  }
  close F;

  for ($i=$first; $i<$last; $i++) {
    system "$grid_work_dir/$i/command_ > $grid_work_dir/$i/c_.log 2>&1 &";
  }
  if ($is_ssh) {
    system "$grid_work_dir/$i/command_ > $grid_work_dir/$i/c_.log 2>&1 &";
  } else {
    # last shell in foreground for SGE
    system "$grid_work_dir/$i/command_ > $grid_work_dir/$i/c_.log 2>&1";
    if ($first != $last) {
      # The last pipe has finished, but those started in background may
      # be active still. So we wait until PIPEDONE_ is to be seen for each of these.
      while (1) {
        for ($i=$first; $i<$last; $i++) {
          last unless -f "$grid_work_dir/$i/PIPEDONE_";
        }
        last if $i == $last;
        sleep 1;
      }
    }
  }
}


sub mergeResults {

  my $jdir = shift;
  my $ftime = shift;           # consider only results newer than that
  $_ = shift;                   # vfiles, common file names in pipes
  my @F = split;
  my $ofn = shift;              # pipe output file

  DeBug("mergeResults of $jdir");
  DeBug("\tofn $ofn\n\t@F", 1);

  my @Logline;

  my ($i,$j, $lastj, $tdir, $fn, $lfn, $mfn, $jfn, $tf, $anyerr, @S, %UsedResult);

  # list of temp. dirs
  for ($j=1; ; $j++) {
    $tdir = "$jdir/$j";
    last unless -d $tdir;
    push @Logline, "\n-------------------- Instance $j --------------------\n\n";
    push @S, $tdir;
    foreach $lfn ("$tdir/c_.log", "$tdir/gridlog") {
      open F, $lfn or next;
      push @Logline, $_ while <F>;
      close F;
      # unlink $lfn;
    }
  }
  my $n = $j - 1;

  # write overall log file
  open OF, ">$jdir/gridlog";
  print OF $_ foreach @Logline;
  close OF;

  foreach $mfn (@F) {

    $tf = "$S[0]/$mfn";     # full path of first file accessed in pipe
    next unless -f $tf;         # no plain file
    next if $ftime >= (stat($tf))[9]; # not modified since job start
    $_ = `file $tf`;
    next unless /ASCII /;       # no text file

    # read first file
    # determine the type spectrum or trajectory file
    # see how many header lines are to be skipped
    # read array @L of lines for spectra

    my ($cols, $skiptext, $err,
        $is_monitor, $is_mergeable_text, $is_xyz, $header_done,
        $weight, $matrix, $m, $tline, @I, @L, @N, @C);
    my $skiplines = 0;
    my $lines = 0;

    open F, $tf;
    while (<F>) {
      chomp;
      unless ($header_done) {
        if (/#/) {
          $skiplines++;
          $skiptext .= "$_\n";
          if (/Trajectories/) {
            $is_mergeable_text = 1;
          } elsif (/Monitor/) {
            $is_monitor = 1;
            $is_xyz = 1 if /x *y *z/;
            $weight = 1 if /weight|mean/;
            $matrix = 1 if /matrix/;
          } elsif (/#x +y +z/) {
            $is_xyz = 1;
          }
          next;
        }
        # we found the first non-comment line
        last if $is_mergeable_text;
        $header_done = 1;
        unless ($is_xyz) {
          # compute the number of columns
          @I = split;
          $cols = 0 + @I;
          $matrix = 1 if $cols > 16;
          if ($matrix) {
            # mon2D format 0
            # skip this line with $cols x-axis tic values
            $skiptext .= "$_\n";
            $skiplines++;
            next;
          } elsif ($cols != 2 && $cols != 4) {
            # unknown spectrum type
            $err = 1;
            last;
          }
        }
      }
      if ($_ eq '') {
        # dummy separation line
        push @L, '';
      } else {
        # normalize numbers, and check if numbers
        @I = ();
        foreach (split) {
          unless (/^[0-9efEF+.-]+$/) {
            # no number
            $err = 1;
            last;
          }
          push @I, 0+$_;
        }
        push @L, "@I";
      }
      $lines++;
    }
    close F;

    if ($is_mergeable_text) {
      # concatenate trajectory files to work dir
      DeBug("writing $jdir/$mfn");
      $UsedResult{$mfn} = 1;
      open OF, ">$jdir/$mfn";
      # copy first file
      open F, $tf;
      print OF $_ while <F>;
      close F;
      for ($j=1; $j<$n; $j++) {
        open F, "$S[$j]/$mfn";
        for ($i=0; $i<$skiplines; $i++) {
          <F>;
        }
        print OF $_ while <F>;
        close F;
      }
      close OF;
      next;                     # file of foreach $mfn loop
    }

    next unless $is_monitor;
    # skip this file silently, if an error occured, assuming it is no spectrum file
    next if $err || $lines < 2;

    # merge contents of following files 2..$n

    DeBug("$lines lines in $tf", 1);
    for ($j=1; $j<$n; $j++) {
      $fn = "$S[$j]/$mfn";
      next unless -s $fn;
      $tline = 0;
      open F, $fn or next;
      for ($i=0; $i<$skiplines; $i++) {
        <F>;
      }
      while (<F>) {
        chomp;
        if ($_ ne '') {
          @N = split;
          last if $tline >= $lines;
          $_ = $L[$tline];
          @I = split;
          if ($matrix) {
            # first column is monitor, followed by $cols counts
            for ($i=1; $i<=$cols; $i++) {
              $I[$i] += $N[$i];
            }
          } elsif ($is_xyz) {
            # ignore x,y
            # increment 3. and 5. column, 4. column error value computed later
            $I[2] += $N[2];
            $I[4] += $N[4];
          } elsif ($cols == 2) {
            $I[1] += $N[1];
          } elsif ($cols == 4) {
            # 2. column weighted count
            $I[1] += $N[1];
            # 3. column error, to be computed later
            # 4. column raw count
            $I[3] += $N[3];
          } else {
            # set anyerr too, to indicate something unexpected
            $err = $anyerr = 1;
            last;
          }
          $L[$tline] = "@I";
        }
        $tline++;
      }
      close F;
      DeBug("read $tline lines in $fn", 1);
      last if $err;
    }

    # do not write this file's result on $err, as file formats differed
    next if $err;

    # print result to work dir
    DeBug("writing $jdir/$mfn");
    $UsedResult{$mfn} = 1;
    open OF, ">$jdir/$mfn";
    print OF $skiptext if $skiptext;
    for ($i=0; $i<$lines; $i++) {
      $_ = $L[$i];
      if ($_ eq '' || !($cols == 4 || $is_xyz || $weight)) {
        # print line without further processing
        print OF "$_\n";
        next;
      }
      # what remains is a   A 1D   B xyz 2D   C to be weighted   spectrum
      @C = split;
      if ($cols == 4) {
        # A      keep 1. column (channel) and 4. column (absolute count)
        if ($C[3] <= 0) {
          print OF "$C[0] 0 0 0\n"; # nothing counted
          next;
        }
        # mean value
        $m = $C[1] / $n;
        # 2. column = mean value  if weight
        $C[1] = $m if $weight;
        # 3. column = statistical error
        $C[2] = $m * sqrt(1.0 / $C[3]);

      } elsif ($is_xyz) {
        # B    x y z 'format'
        # keep 1. column (x), 2. column (y) and 5. column (absolute count)
        if ($C[4] <= 0) {
          print OF "$C[0] $C[1] 0 0 0\n"; # nothing counted
          next;
        }
        # mean value
        $m = $C[2] / $n;
        # 3. column = mean value   if weight
        $C[2] = $m if $weight;
        # 4. column = statistical error
        $C[3] = $m * sqrt(1.0 / $C[4]);

      } else {
        # C    normalize counts to average
        if ($matrix) {
          # 2D spectrum
          my $k;
          # again: $C[0] is y axis tic-value, followed by $cols counts
          for ($k=1; $k<=$cols; $k++) {
            $C[$k] /= $n;
          }
        } elsif ($cols == 2) {
          $C[1] /= $n;
        }
      }
      print OF "@C\n";
    }
    close OF;

  }

  if ($verbose > 1) {
    # show which new files have not been merged
    my @NotProcessed;
    opendir D, $S[0];
    while ($_ = readdir D) {
      next if /^\./;
      next if $UsedResult{$_};
      next if (stat "$S[0]/$_")[9] <= $ftime;
      push @NotProcessed, $_;
    }
    closedir D;
    print "result files _not_ processed @NotProcessed\n" if @NotProcessed;
  }

  # move geometry files to the working directory - they are the same in all instance dirs
  foreach (qw(instrument.inf simulation.inf geometry.inf)) {
    rename "$S[0]/$_", "$jdir/$_";
  }

  return $anyerr unless $ofn;

  # concatenate output of last module, if any
  $_ = "$jdir/$n/$ofn";
  DeBug("look at outfile $_", 1);
  return 1 unless -s $_;

  # we have some neutron trajectory output
  if ($n <= 1) {
    # just move that file to the work directory
    rename $_, "$jdir/$ofn";
  } else {
    # concatenate files
    open OF, ">$jdir/$ofn";
    my ($r, $buffer);
    for ($j=1; $j<=$n; $j++) {
      $jfn = "$jdir/$j/$ofn";
      open F, $jfn;
      while (($r = read F, $buffer, 4096*120) > 0) {
        syswrite OF, $buffer, $r;
      }
      close F;
      unlink $jfn;              # to keep quota a bay
    }
    close OF;
  }
  0;                            # everthing's fine

}

sub myexit {
  print "$_[0]!\n";
  exit 2;
}

sub DeBug {
  return unless $verbose;
  if ($verbose < 2) {
    print "$_[0]\n" unless $_[1];
  } else {
    $_ = time - $start_time;
    print "DEBUG $_ s: $_[0]\n";
  }
}

#############################################################################################
### routines for optimization runs

sub parseOptInput {
  my (@I, @P);

  # sim_param.ini
  open F, $sini_file or myexit('unable to read sim ini file');
  my $lc = 0;
  while (<F>) {
    next if /^\#/;              # ignore comment lines
    chomp;
    s/\#.*//;                   # ignore rest comments
    s/\s+$//;                   # ignore trailing white space
    next unless $_;
    if ($lc > 3) {
      @I = split;
      push @SimPars, "@I";
    } elsif ($lc == 0) {
      $opt_rdir = $_ if $_ ne 'STD';
    } elsif ($lc == 1) {
      $opt_pdir = $_;
      myexit("directory $opt_pdir does not exist") unless -d $opt_pdir;
    } elsif ($lc == 2) {
      @FileOfMerit = split;
    } elsif ($lc == 3) {
      # don't know
    }
    $lc++;
  }
  close F;

  # Pcomm.dat
  open F, $opt_dat or myexit("unable to read sim dat file $opt_dat");
  <F>;                          # skip first line
  while (<F>) {
    chomp;
    @I = split;
    push @SimDat, "@I";
  }
  close F;
  $n_instances = 0 + @SimDat;

  # std_instr.cmd

  open F, $instr_file or myexit('unable to read sim cmd file');
  while (<F>) {
    next unless /--/; # skip anything but commands, which have -- options
    chomp;
    tr/\\/\//;                  # if something came for windoof
    if ($opt_pdir ne './') {
      s/--P$opt_pdir/--P\$P/g;  # remove explict parameter directory
      s/$opt_pdir/\$P/g;
    }
    @I = split;
    $_ = shift @I;
    $_ = shift @I if $_ eq '|';      # remove pipe bar
    $_ = $1 if m@([a-zA-Z0-9._-]+)$@; # sanitize image name
    push @Pipe, "\$V/$_ @I";
  }
  close F;

  # identify parameters to be replaced later
  my $count = 0;
  foreach (@SimPars) {
    my ($n, $p) = split;
    $Pipe[$n-1] =~ s/ $p[^ ]+/ ${p}VVV${count}VVV/;
    $count++;
  }

  analyzePipe($opt_rdir, $opt_pdir);

}

sub pollExecutionState {

  my ($i, @Pipedone);

  # poll the first pipe if it's finished
  my $tfile = "$grid_work_dir/1/PIPEDONE_";
  while (1) {
    last if -f $tfile;
    sleep 2;
  }

  # poll other pipes, with a higher frequency
  while (1) {
    my $anyleft = 0;
    for ($i=2; $i <= $n_instances; $i++) {
      next if $Pipedone[$i];
      if (-f "$grid_work_dir/$i/PIPEDONE_") {
        $Pipedone[$i] = 1;
      } else {
        $anyleft = 1;
      }
    }
    last unless $anyleft;
    sleep 1;
  }
}

sub deleteOldFOMFiles {
  my ($n, @F);
  foreach $n (@FileOfMerit) {
    # split name to filename + extension
    # delete <filename>*.<extension>
    next unless $n =~ /^(.+)\.([^.]+)$/;
    (@F) = glob "$1*.$2";
    next unless @F;
    unlink @F;
    DeBug("unlink @F", 1);
  }
}

sub extractFigureOfMeritFiles {
  my ($i, $n, $from, $to);
  for ($i=1; $i <= $n_instances; $i++) {
    foreach $n (@FileOfMerit) {
      $from = "$grid_work_dir/$i/$n";
      next unless -f $from;
      # split name to filename + extension
      $n =~ /^(.+)\.([^.]+)$/;
      # example sample.mtl will be sample1.mtl
      $to = "$1$i.$2";
      copy($from, $to);
      DeBug("copy $from, $to", 1);
    }
  }
}


sub doOptimization {

  # set some defaults, if either slnodes or slpartition is given
  if ($slpartition ne '' && $slnodes eq '') {
    $slnodes = 1;
    $submit_mode = 'slurm';
  } elsif ($slnodes ne '' && $slpartition eq '') {
    $slpartition = 'long';
  }
  $jobname = 'optrun' if $jobname eq '';
  $_ = $ENV{GRIDRUNDIR};
  $dir_pattern = /^[a-v]$/ ? $_ : 'o';

  &parseOptInput;

  &createWorkdir;

  &checkSSH;

  &deleteOldFOMFiles;

  &startExecution;

  &pollExecutionState;

  &extractFigureOfMeritFiles;

  # mark work directory to be re-used later
  open F, ">$grid_work_dir/FINISHED_";
  close F;

  DeBug('gridrun opt finished');

}
