#!/bin/csh -f
# Copyright(C) 1999-2020 National Technology & Engineering Solutions
# of Sandia, LLC (NTESS).  Under the terms of Contract DE-NA0003525 with
# NTESS, the U.S. Government retains certain rights in this software.
#
# See packages/seacas/LICENSE for details
#**************************************************************************
#**************************************************************************
#**
#**  SCRIPT: loadbal
#**
#**************************************************************************
#**************************************************************************

# Determine ACCESS based on the location of the script
if ("`echo $0 | sed 's/\///'`" != "$0") then
# Location of the script found
   if ("`ls -l $0 | sed 's/^l.*/l/'`" != "l") then
# Script is not a softlink
      setenv ACCESS `echo $0 | sed 's/\/etc\/.*//'`
      if (! -e $ACCESS/config/site.def) then
         setenv ACCESS @ACCESSDIR@
      endif
   else
# Script is a softlink
      setenv ACCESS @ACCESSDIR@
   endif
else
# Determining the location of the script failed, use default values
   setenv ACCESS @ACCESSDIR@
endif

# initialize variables

echo "==========================================================="
echo "| Sandia Engineering Analysis Access procedure for: loadbal"
echo '| $Revision: 1.57 $, $Date: 2008/05/19 12:29:22 $'
echo "==========================================================="
set codename=loadbal
set runid=DEFAULT
set mesh_opt=DEFAULT
set machine=DEFAULT
set decomp_method="-l inertial"
set nem_slice_flag="-c"
set pexmach=DEFAULT
set rootdir=DEFAULT
set subdir=DEFAULT
set offset=DEFAULT
set raids=DEFAULT
set scalar_mesh="yes"
set subdirectory="yes"

set userid = `echo $LOGNAME`
set curdir = `pwd`

set ACCESSBIN = $ACCESS/bin
set NEM_SLICE = $ACCESSBIN/nem_slice

set default_raids  = 1
set default_root   = `pwd`/
set default_sub    = .
set default_offset = 1

set input_suffix   = i
set genesis_suffix = g
set exodus_suffix  = e
set spread_suffix  = par
set spreadit       = "NO"

env >! JUNK
grep "ALEGRA_EXE" JUNK > JUNK2
set alegra_rc = $status
if($alegra_rc == 0) then
  set input_suffix   = inp
  set genesis_suffix = gen
  set exodus_suffix  = exo
endif
/bin/rm -rf JUNK JUNK2

# Parse command line so that it can be used by the options program
if ($#argv == 0) goto USAGE
if ($#argv == 1 && "$argv[1]" == "-h") goto USAGE
@ i = 1
while ($i < $#argv)
    switch ($argv[$i])
    case -h:
        set argv[$i] = "-help"
        breaksw
    case -n*:
        set argv[$i] = "-nem_slice_flag"
        @ i++
        set testvar = `echo $argv[$i] | sed "s/.*\-l/\-l/" | sed "s/\-l.*//"`
        if ( "$testvar" == "" ) then
            set decomp_method = "$argv[$i]"
        else
            set nem_slice_flag = "$argv[$i]"
        endif
        set argv[$i] = "user_defined"
        breaksw
    case -o:
        set argv[$i] = "-offset"
        breaksw
    case -p:
        set argv[$i] = "-mesh_opt"
        breaksw
    case -r:
        set argv[$i] = "-raids"
        breaksw
    case -R:
        set argv[$i] = "-ROOTDIR"
        breaksw
    case -S:
        set argv[$i] = "-SUBDIR"
        breaksw
    default:
        breaksw
    endsw
    @ i++
end

# Parse command line
set argv  = (`$ACCESS/bin/options $*`)

# Set options exit condition
set errflg = $status
# echo command line input
echo "Input options: $argv"
# Check errflg - nonzero indicates error in options, print usage and exit:
if ($errflg != 0) then
        goto USAGE
endif
# Process options for this code
# Check for basename argument (Will follow "--" in argument list)
set temp = ($argv)
while ("$temp[1]" != "--")
        shift temp
end
# temp points to --
shift temp
# Set the basename
if ($#temp <= 0) then
        echo "ERROR:******************************************************************"
        echo "ERROR:******************************************************************"
        echo "ERROR:*************         No run id specified            *************"
        echo "ERROR:******************************************************************"
        echo "ERROR:******************************************************************"
        set errflg = 1
        goto USAGE
endif
set runid = $temp[1]
# Check for arguments and set corresponding variables
while ("$argv[1]" != "--")
        switch ($argv[1])
        case -help:
                goto USAGE
                breaksw
        case -nem_slice_flag:
                shift argv
                breaksw
        case -offset:
                shift argv
                set offset = $argv[1]
                breaksw
        case -mesh_opt:
                shift argv
                set mesh_opt = $argv[1]
                breaksw
        case -raids:
                shift argv
                set raids = $argv[1]
                breaksw
        case -ROOTDIR:
                shift argv
                set rootdir = $argv[1]
                breaksw
        case -SUBDIR:
                shift argv
                set subdir = $argv[1]
                breaksw
        case -spread:
	        set spreadit="yes"
		breaksw
        case -slit:
	        set nem_slice_flag = "-c"
		breaksw
        case -suffix_mesh:
                shift argv
                set genesis_suffix = $argv[1]
                breaksw
        case -suffix_spread:
                shift argv
                set spread_suffix = $argv[1]
                breaksw
        case -suffix_transient:
                shift argv
                set exodus_suffix = $argv[1]
                breaksw
        case -No_scalar_mesh:
	        set scalar_mesh = "no"
		breaksw
        case -No_subdirectory:
	        set subdirectory = "no"
		breaksw
	case -No_01:
	        set subdirectory = "no"
		breaksw
        case -multikl:
	        set decomp_method="-l multikl"
                breaksw
        case -rcb:
	        set decomp_method="-l rcb"
		breaksw
        case -rib:
	        set decomp_method="-l rib"
		breaksw
        case -hsfc:
	        set decomp_method="-l hsfc"
		breaksw
	case -spectral:
	        set decomp_method="-l spectral"
                breaksw
	case -inertial:
	        set decomp_method="-l inertial"
                breaksw
	case -linear:
	        set decomp_method="-l linear"
                breaksw
	case -random:
	        set decomp_method="-l random"
                breaksw
	case -scattered:
	        set decomp_method="-l scattered"
                breaksw
	case -brick:
	        set decomp_method="-l brick"
		breaksw
	case -zpinch:
	        set decomp_method="-l zpinch"
		breaksw

        default:
                echo "ERROR: unrecognized option $argv[1]"
                goto USAGE
                breaksw
        endsw
        shift argv
end

if($mesh_opt == DEFAULT) then
  echo
  echo "ERROR:***************************************************"
  echo "ERROR:***************************************************"
  echo "ERROR:>>>>  Must specify -p option (number of processors)."
  echo "ERROR:***************************************************"
  echo "ERROR:***************************************************"
  echo
  goto USAGE
endif

if($runid == DEFAULT) then
  echo runid unknown. Specify an file prefix to run.
  goto USAGE
else
  set input   = $runid.$input_suffix
  set output  = $runid.lbd.out
  set error   = $runid.lbd.err
  set nemesis = $runid.nem
  set genesis = $runid.$genesis_suffix
  set exodus  = $runid.$exodus_suffix
  set pexsh   = $runid.pex
  set spreed  = $runid.spd
  set config  = $runid.cfg
  goto EXECUTE_LBD
endif

# *************************************************************************
# *************************************************************************

EXECUTE_LBD:

echo ""
echo "***************************************************"
echo "*"
echo "*    Starting loadbal calculation for $runid"
echo -n "*         at "; date
echo "*"
echo "*    .... beginning decomposition ...."
echo "*         see $output for current status"



echo $mesh_opt >! DUMMY
sed -e 's/\x/\*/g' < DUMMY >! DUMMY2
sed -e 's/\x/\ /g' < DUMMY >! DUMMY3
set procopt = `cat DUMMY3`
echo "quit" >> DUMMY2
/usr/bin/bc DUMMY2 >! DUMMY
set numproc = `cat DUMMY`
/bin/rm -rf DUMMY DUMMY2 DUMMY3

set date = `date '+%m/%d/%y'`
set time = `date '+%H:%M:%S'`
set prob_dir = `pwd | sed "s/.*\///g"`

echo "===========================================================" >! $output
echo "| Sandia Engineering Analysis Access procedure for: loadbal" >> $output
echo '| $Revision: 1.57 $, $Date: 2008/05/19 12:29:22 $'                                      >> $output
echo "===========================================================" >> $output

echo "Executing:"
echo "($NEM_SLICE -e $decomp_method $nem_slice_flag -o $nemesis -m mesh=$mesh_opt -a $input $genesis"
if(-f $input) then
($NEM_SLICE -e $decomp_method $nem_slice_flag -o $nemesis -m mesh=$mesh_opt -a $input $genesis >> $output) >& $error
set load_rc = $status
else
($NEM_SLICE -e $decomp_method $nem_slice_flag -o $nemesis -m mesh=$mesh_opt $genesis >> $output) >& $error
set load_rc = $status
endif

if($load_rc != 0) then
  goto BOMBED
else
  goto LAN
endif

# *************************************************************************
# *************************************************************************

CONFIG:

echo "set machine     = $machine"                           >! $config
echo "set proc_conf   = $mesh_opt"                          >> $config
echo "set num_procs   = $numproc"                           >> $config
echo "set root_dir    = $rootdir"                           >> $config
echo "set sub_dir     = $subdir"                            >> $config
echo "set job_name    = $runid"                             >> $config
echo "set num_raids   = $raids"                             >> $config
echo "set raid_offset = $offset"                            >> $config
echo "set nem_date    = $date"                              >> $config
echo "set nem_time    = $time"                              >> $config
echo "set zero = 0"                                         >> $config

chmod 755 $spreed
set mpexe = local

if($spreadit == "yes") then
  echo ""
  echo "Executing: ./$spreed  ..."
  (./$spreed  >> $output) >& $error
endif

echo "*"
echo "*    Completed loadbal calculation for $runid"
echo -n "*         at "; date
echo "*"
echo "*    For details of the decomposition, see"
echo "*        $output (output) and"
echo "*        $error (possible errors/warnings)"
echo "*"
echo "*    The following files may be required for parallel runs:"
echo "*          $input"
echo "*          $genesis"
echo "*          $nemesis"
echo "*          $spreed"
echo "*          $config"
echo "*"
echo "***************************************************"
echo " "

if ($?SEACAS_NO_LOGGING) then
  echo "SEACAS Audit logging disabled via SEACAS_NO_LOGGING setting"
else

# Echo audit information to ${ACCESS}/etc/audit.log
set time_used=`time`

# Get name of user running code
if (${?USER}) then
	set username = ${USER}
else
	if (${?LOGNAME}) then
		set username = ${LOGNAME}
	else
		set username = "UNKNOWN"
	endif
endif

# Write info to audit.log file
if (-w ${ACCESS}/etc/audit.log) then
  echo ${codename} ${username} `date` ${time_used} `hostname` >>${ACCESS}/etc/audit.log
endif
endif

exit $load_rc

# *************************************************************************
# *************************************************************************

LAN:

if($rootdir == DEFAULT) then
  set rootdir=$default_root
endif

if($subdir == DEFAULT) then
  set subdir=$default_sub
endif

if($offset == DEFAULT) then
  set offset=$default_offset
endif

if($raids == DEFAULT) then
  set raids=$default_raids
endif

if($raids > $numproc) then
  set raids = $numproc
endif

set pexcmd  = $ACCESSBIN/nem_spread

if ($subdirectory == "no") then
  set do_sub = ", nosubdirectory"
else
  set do_sub =
endif

set zero = "zeros"
set stage =

if(-f $pexsh) then
  /bin/rm -rf $pexsh
endif

echo "Creating $pexsh"
echo "Input FEM file                   = $genesis"                  >! $pexsh
echo "LB file                          = $nemesis"                  >> $pexsh
echo "File Extension for Spread Files  = .$spread_suffix"           >> $pexsh
echo "Parallel Results file base name  = $exodus "                  >> $pexsh
echo "Number of Processors             = $numproc "                 >> $pexsh
echo "Use Scalar Mesh File             = $scalar_mesh "             >> $pexsh
echo "------------------------------------------------------------" >> $pexsh
echo "                Parallel I/O section"                         >> $pexsh
echo "------------------------------------------------------------" >> $pexsh
echo "Parallel Disk Info= number=$raids, offset=$offset $stage, $zero $do_sub" >> $pexsh
echo "Parallel file location = root=$rootdir, subdir=$subdir"       >> $pexsh

if(-f $spreed) then
  /bin/rm -rf $spreed
endif

echo "Creating $spreed"
echo "#\!/bin/csh -f"                                               >! $spreed
echo ""                                                             >> $spreed
if ($subdirectory == "yes") then
  echo "set root = $rootdir"                                        >> $spreed
  echo "set subdir = $subdir"                                       >> $spreed
  echo "set numraid = $raids"                                       >> $spreed
  echo "set i = $offset"                                            >> $spreed
  echo "set j = 1"                                                  >> $spreed
  echo "set stride = 1"                                             >> $spreed
  echo "set zero = 0"                                               >> $spreed
  echo 'while($j <= $numraid)'                                      >> $spreed
  echo '  if($i < 10) then'                                         >> $spreed
  echo '    set drive = $root$zero$i'                               >> $spreed
  echo '  else'                                                     >> $spreed
  echo '    set drive = $root$i'                                    >> $spreed
  echo '  endif'                                                    >> $spreed
  echo '  set dir = $drive/$subdir'                                 >> $spreed
  echo '  if(!(-d $dir)) then'                                      >> $spreed
  echo '    echo "making directory $dir"'                           >> $spreed
  echo '    mkdir -p $dir'                                          >> $spreed
  echo '  endif'                                                    >> $spreed
  echo '  @ i += $stride'                                           >> $spreed
  echo '  @ j++'                                                    >> $spreed
  echo 'end'                                                        >> $spreed
else
  if ($subdir != $default_sub) then
    echo "set root = $rootdir"                                      >> $spreed
    echo "set subdir = $subdir"                                     >> $spreed
    echo 'set dir = $root/$subdir'                                  >> $spreed
    echo 'if(!(-d $dir)) then'                                      >> $spreed
    echo '  echo "making directory $dir"'                           >> $spreed
    echo '  mkdir -p $dir'                                          >> $spreed
    echo 'endif'                                                    >> $spreed
  endif
endif
echo ""                                                             >> $spreed
echo "$pexcmd $pexsh"                                               >> $spreed

goto CONFIG

# *************************************************************************
# *************************************************************************

BOMBED:

echo ""
echo ""
echo "ERROR:***************************************************"
echo "ERROR:***************************************************"
echo "ERROR:>>>>   loadbal exiting with status: $load_rc"
echo "ERROR:>>>>   This suggests that a problem occurred."
echo "ERROR:>>>>   The output of $error is:"
echo "ERROR:***************************************************"
echo "ERROR:***************************************************"
echo ""
cat $error
echo ""
exit $load_rc

# *************************************************************************
# *************************************************************************

USAGE:
cat << 'EOF'

Usage:  loadbal [-h] -p <processor layout> [-n <nemslice options>]
                [-r <number of disks>] [-o <disk offset>] [-slit]
                [-R <root directory>] [-S <sub directory] [-spread]
                [-multikl] [-spectral] [-inertial] [-linear]
                [-rcb] [-rib] [-hsfc] [-random] [-scattered]
                [...others...] meshfile_basename

Decomposition methods: [0..3] = speed ranking 0=fastest
  -multikl:   [3] Multilevel KL. Gives high quality partitions.
		  Slow, high memory use.
  -spectral:  [2] use eigenvectors of a matrix constructed from the
                  graph to partition the graph.
  -hsfc       [?] Hilbert Space-Filling Curve (uses Zoltan)
  -rib:       [?] Recursive Inertial Bisection (uses Zoltan)
  -rcb:       [?] Recursive Coordinate Bisection (uses Zoltan)
  -inertial:  [1] (default) vertices divided into sets of equal mass by planes
                  orthogonal to the principle axis. Good, fast, medium memory
  -linear:    [0] vertices in order first n/p to proc 0, next to proc 1.
                  (fast and sometimes good).
  -random:    [0] vertices are assigned randomly to sets in a way that
                  preserves balance.
  -scattered: [0] vertices are handed out in order with the next vertex
                  going to whichever set is smallest.
  -brick      [-] special decomposition method; not useful for general use.
  -zpinch     [-] special decomposition method; not useful for general use.
      (See jal.sandia.gov/SEACAS/Documentation/chaco.pdf and
           www.cs.sandia.gov/Zoltan/ug_html/ug.html for details)

Other options:
-slit           Do not flag error if mesh contains slit/crack/sipe
                (two quad faces sharing 3 nodes) [Set by default now]
-spread         Execute the {runid}.spd file automatically as part of the
                loadbal process.
-h              print this message
-n options      options for nem_slice. Example: -n -c
-o offset       disk drive number to begin with
-p proc_layout  select a one-, two-, three-dimensional mesh layout
                Examples: -p 4       a one-dimensional mesh of 4 processors
                          -p 4x4     a two-dimensional mesh of 16 processors
                          -p 3x4x2   a three-dimensional mesh of 24 processors
-r num_disks    number of separate disk drives to use
-R root_dir     root directory to begin the path to on the separate disks
-S sub_dir      continuation of the path on the separate disks to the files
-suffix_mesh suf      file extension on input mesh file (default = g)
-suffix_spread suf    file extension on spread files (default = par)
-suffix_transient suf file extension on results files (default = e)
-No_scalar_mesh       Do not use the scalar mesh in nem_join;
                      use only results files.
-No_subdirectory      Do not create the 01 subdirectory;
                      write files in root_dir/sub_dir.

   help-email: SEACAS@-HELPsandia.gov (code, script)
  Information: www.jal.sandia.gov/SEACAS/SEACAS.html (IRN)
               ${ACCESS}/doc/index.html     (local)
'EOF'
exit 11