EXODUS/packages/seacas/scripts/decomp.in

#!/bin/bash
# Copyright(C) 1999-2020, 2023 National Technology & Engineering Solutions
# of Sandia, LLC (NTESS).  Under the terms of Contract DE-NA0003525 with
# NTESS, the U.S. Government retains certain rights in this software.
#
# See packages/seacas/LICENSE for details
########################################################################
function usage {
  if [ $OLDGETOPT -eq 1 ] ; then
    usage_old
  else
    usage_new
  fi
}

function usage_new {
cat <<DECOMP_USAGE_EOF
Usage:  decomp [-h] --processors <processor count> [-n <nemslice options>]
               [--root <root directory>] [--subdir <sub directory>] [--spread_suffix <suffix>]
               [--multikl] [--spectral] [--inertial] [--linear]
               [--rcb] [--rcb_ignore_z] [--rib] [--hsfc] [--random] [--scattered]
               [--weight <specification>] [...others...] meshfile.ext

Decomposition methods: [0..3] = speed ranking 0=fastest
              (See chaco and/or zoltan documentation for more details)
 --multikl:   [3] Multilevel KL. Gives high quality partitions.
                  Slow, high memory use.
 --spectral:  [2] use eigenvectors of a matrix constructed from the
                  graph to partition the graph.
 --hsfc       [?] Hilbert Space-Filling Curve (uses Zoltan)
 --rib:       [?] (default) Recursive Inertial Bisection (uses Zoltan)
 --rcb:       [?] Recursive Coordinate Bisection (uses Zoltan)
 --rcb_ignore_z:       [?] Recursive Coordinate Bisection (uses Zoltan), ignoring the z dimension coordinates
 --inertial:  [1] vertices divided into sets of equal mass by planes
                  orthogonal to the principle axis. Good, fast, medium memory
 --linear:    [0] vertices in order first n/p to proc 0, next to proc 1.
                  (fast and sometimes good).
 --random:    [0] vertices are assigned randomly to sets in a way that
                  preserves balance.
 --scattered: [0] vertices are handed out in order with the next vertex
                  going to whichever set is smallest.
 --brick      [-] special decomposition method; not useful for general use.
 --zpinch     [-] special decomposition method; not useful for general use.

 NOTE: The decomposition options can also be specified as "-l type"
       For example "--multikl" is the same as "-l multikl"

Other options:
--help              Print this message (-h)
--noop              Only echo the nem_slice and nem_spread commands.
--nodal             Nodal Decomposition (-N)
--elemental         Elemental Decomposition (default, -E)
--input <file>      Optional nem_slice input file (-i)
--weight  <spec>    Specification of weighting options (-w)
--viz_mesh          Output mesh showing processor distribution of elements. (-V)
--spheres_linear    Decompose sphere elements using linear method (matches old behavior) (-s)
--processors <#p>   Specify number of processors (-p, -j)
--root <root_dir>   Root directory to begin the path to on the separate disks (-R)
--subdir <sub_dir>  Continuation of the path on the separate disks to the files (-S)
--spread_suffix <x> Suffix for parallel files (default: meshfile suffix) (-X)
--64                Force creation of 64-bit integer decomposed files (-6)
-n <options>        Specify additional nem_slice options

   Documentation: https://sandialabs.github.io/seacas-docs/sphinx/html/index.html#decomp"
   help-email: sierra-help@sandia.gov

DECOMP_USAGE_EOF
exit 1
}

function usage_old {
cat <<DECOMP_USAGE_EOF
Usage:  decomp [-h] -p <processor count> [-n <nemslice options>]
               [-R <root directory>] [-S <sub directory>] [-X <suffix>]
               [-l multikl] [-l spectral] [-l inertial] [-l linear]
               [-l rcb] [-l rcb_ignore_z] [-l rib] [-l hsfc] [-l random] [-l scattered]
               [...others...] meshfile.ext

Decomposition methods: [0..3] = speed ranking 0=fastest
              (See chaco and/or zoltan documentation for more details)
 -l multikl:   [3] Multilevel KL. Gives high quality partitions.
                  Slow, high memory use.
 -l spectral:  [2] use eigenvectors of a matrix constructed from the
                  graph to partition the graph.
 -l hsfc       [?] Hilbert Space-Filling Curve (uses Zoltan)
 -l rib:       [?] (default) Recursive Inertial Bisection (uses Zoltan)
 -l rcb:       [?] Recursive Coordinate Bisection (uses Zoltan)
 -l rcb_ignore_z:       [?] Recursive Coordinate Bisection (uses Zoltan) ignoring the z dimension coordinates
 -l inertial:  [1] vertices divided into sets of equal mass by planes
                  orthogonal to the principle axis. Good, fast, medium memory
 -l linear:    [0] vertices in order first n/p to proc 0, next to proc 1.
                  (fast and sometimes good).
 -l random:    [0] vertices are assigned randomly to sets in a way that
                  preserves balance.
 -l scattered: [0] vertices are handed out in order with the next vertex
                  going to whichever set is smallest.
 -l brick      [-] special decomposition method; not useful for general use.
 -l zpinch     [-] special decomposition method; not useful for general use.

Other options:
-N          Nodal Decomposition
-E          Elemental Decomposition (default)
-n options  Specify nem_slice options
-h          Print this message
-i          Optional nem_slice input file
-s          Decompose sphere elements using linear method (matches old behavior)
-p #p       Specify number of processors (or -j)
-R root_dir Root directory to begin the path to on the separate disks
-S sub_dir  Continuation of the path on the separate disks to the files
-X suffix   Suffix for parallel files (default: meshfile suffix)
-V          Output mesh showing processor distribution of elements
-6          Force creation of 64-bit integer decomposed files

   Documentation: https://sandialabs.github.io/seacas-docs/sphinx/html/index.html#decomp"
   help-email: sierra-help@sandia.gov

DECOMP_USAGE_EOF
exit 1
}

function old_getopt {
echo ${txtred}
cat <<DECOMP_EOF
########################################################################
The "getopt" executable that is available on this system is an older
version that is not compatible with the needs of the "decomp" tool.
If possible, you should update your getopt to a newer version and make
sure that the new getopt is in your path.

Below are some options for getting the current getopt version:
* If on a Mac: "sudo port install getopt" or "brew install gnu-getopt"
* Search the internet for "getopt-1.1.5" or "getopt-1.1.4"; download and build

Enter "-h" for the modified options that this version supports.
Enter "-H" for the options that the standard version supports.
########################################################################

DECOMP_EOF
echo ${txtrst}
}

########################################################################
function execute_loadbalance {
    basename=$1
    processors=$2
    decomp_method=$3
    nem_slice_flag=$4
    NEM_SLICE=$5
    input=$6
    decomp_type=$7
    weighting=$8

    nemesis=$basename.$suffix_mesh.nem
    genesis=$basename.$suffix_mesh

    # Check for valid executable...
    if [ ! -x $NEM_SLICE ]
    then
        echo
        echo ${txtred}
        echo "ERROR: Could not find or execute $NEM_SLICE"
        echo "ERROR: Configuration error"
        echo ${txtrst}
        echo

        return -1
    fi

    if [ -n "$input" ] && [ -e "$input" ]
    then
        echo ${txtblu}
        echo "Executing:"
        echo "   $NEM_SLICE $decomp_type $spheres $do_viz $decomp_method $weighting $nem_slice_flag -o $nemesis -m mesh=$processors -a $input $genesis"
        echo ${txtrst}
        (${NOOP:+echo }$NEM_SLICE $decomp_type $spheres $do_viz $decomp_method $weighting $nem_slice_flag -o $nemesis -m mesh=$processors -a $input $genesis)
        load_rc=$?
    else
        echo ${txtblu}
        echo "Executing:"
        echo "   $NEM_SLICE $decomp_type $spheres $do_viz $decomp_method $weighting $nem_slice_flag -o $nemesis -m mesh=$processors $genesis"
        echo ${txtrst}
        (${NOOP:+echo }$NEM_SLICE $decomp_type $spheres $do_viz $decomp_method $weighting $nem_slice_flag -o $nemesis -m mesh=$processors $genesis)
        load_rc=$?
    fi
    return $load_rc
}

########################################################################
# Create the nem_spread input file (basename.pex) and execute nem_spread...
function execute_spread {
    basename=$1
    rootdir=$2
    subdir=$3
    numproc=$4
    NEM_SPREAD=$5

    nemesis=$basename.$suffix_mesh.nem
    genesis=$basename.$suffix_mesh

    pexsh=$basename.$suffix_mesh.pex

    # Check for valid executable...
    if [ ! -x $NEM_SPREAD ]
    then
        echo
        echo ${txtred}
        echo "ERROR: Could not find or execute $NEM_SPREAD"
        echo "ERROR: Configuration error"
        echo ${txtrst}
        echo
        return -1
    fi

    if [ "$subdir" != "." ]
    then
        dir=$rootdir/$subdir
        if [ ! -e $dir ]
        then
            echo "making directory $dir"
            mkdir -p $dir
        fi
    fi

    if [ -e $pexsh ]
    then
        /bin/rm -rf $pexsh
    fi

    echo "Input FEM file                   = $genesis"                  >  $pexsh
    echo "LB file                          = $nemesis"                  >> $pexsh
    echo "Parallel Results File Base Name  = $basename"                 >> $pexsh
    echo "File Extension for Spread Files  = .$suffix_spread"           >> $pexsh
    echo "Number of Processors             = $numproc "                 >> $pexsh
    if [ "$debug_level" != "0" ]
    then
    echo "Debug                            = $debug_level"              >> $pexsh
    fi
    echo "------------------------------------------------------------" >> $pexsh
    echo "                Parallel I/O section"                         >> $pexsh
    echo "------------------------------------------------------------" >> $pexsh
    echo "Parallel Disk Info= number=1, offset=1, zeros, nosubdirectory">> $pexsh
    echo "Parallel file location = root=$rootdir, subdir=$subdir"       >> $pexsh

    echo ${txtblu}
    echo "Executing:"
    echo "   $NEM_SPREAD $use64 $pexsh"
    echo ${txtrst}
    ${NOOP:+echo }$NEM_SPREAD $use64 $pexsh
}

########################################################################
# initialize variables
# Text color variables
if [[ $TERM == *"xterm"* ]] || [[ $TERM == "screen" ]]; then
    txtund=$(tput sgr 0 1)    # Underline
    txtbld=$(tput bold)       # Bold
    txtred=$(tput setaf 1)    # Red
    txtgrn=$(tput setaf 2)    # Green
    txtylw=$(tput setaf 3)    # Yellow
    txtblu=$(tput setaf 4)    # Blue
    txtpur=$(tput setaf 5)    # Purple
    txtcyn=$(tput setaf 6)    # Cyan
    txtwht=$(tput setaf 7)    # White
    txtrst=$(tput sgr0)       # Text reset
else
    export TERM=dumb
    txtund=""
    txtbld=""
    txtred=""
    txtgrn=""
    txtylw=""
    txtblu=""
    txtpur=""
    txtcyn=""
    txtwht=""
    txtrst=""
fi

basename=''
processors=''
debug_level='0'
decomp_type="-e"
decomp_method="-l rib"
nem_slice_flag="-c"
weighting=''
rootdir="$(pwd)/"
subdir="."
suffix_spread=''
spheres="-S"
do_viz=""
use64=""

NOOP=

if [ "$ACCESS" == "" ]
then
    ACCESS=@ACCESSDIR@
fi

# See if the path to the script is in ${ACCESS}/bin/decomp.
pushd $(dirname "${0}") > /dev/null
basedir=$(pwd -P)
popd > /dev/null
pushd ${ACCESS}/bin > /dev/null
instdir=$(pwd -P)
popd > /dev/null
# Use "pwd -P" for the path without links. man bash for more info.

# Several options for execution:
# 1. If nem_slice and nem_spread are in same directory as script, use that path.
# 2. Check location specified by ${ACCESS} environment variable to see if decomp, nem_slice, nem_spread
#   -- If ACCESS specified by environment variable, else
#   -- Use hard-wired value above.
# 3. If not, then assume we are running in the cmake build directory
#    and update the paths to nem_slice and nem_spread accordingly...

if [ -x ${basedir}/nem_slice ] && [ -x ${basedir}/nem_spread ]; then
  NEM_SLICE=${basedir}/nem_slice
  NEM_SPREAD=${basedir}/nem_spread
elif [ "${basedir}" == "${instdir}" ] ; then
  NEM_SLICE=${ACCESS}/bin/nem_slice
  NEM_SPREAD=${ACCESS}/bin/nem_spread
else
  NEM_SLICE=${basedir}/../applications/nem_slice/nem_slice
  NEM_SPREAD=${basedir}/../applications/nem_spread/nem_spread
fi

########################################################################
# Test that the getopt being used will handle the long options...
TEST=$(getopt -T)
if [ $? != 4 ] ; then
  old_getopt
  OLDGETOPT=1
else
  OLDGETOPT=0
fi

if [ $# -eq 0 ] ; then
    usage
fi

########################################################################
# decomp options:
if [ $OLDGETOPT -eq 1 ] ; then
   TEMP=$(getopt Hh6ENn:i:o:p:j:r:R:S:X:sl:vVd: "$@")
else
   TEMP=$(getopt -o Hh6ENn:i:o:p:j:r:R:S:X:sl:vVd:w: -a \
       --long input: \
       --long noop \
       --long help \
       --long nem_slice_flag \
       --long 64 \
       --long processors: \
       --long rootdir: \
       --long subdir: \
       --long spread_suffix: \
       --long spheres_linear \
       --long viz_mesh \
       --long verbose \
       --long debug: \
       --long nodal \
       --long elemental \
       --long multikl \
       --long rcb \
       --long rcb_ignore_z \
       --long rib \
       --long hsfc \
       --long spectral \
       --long inertial \
       --long linear \
       --long random \
       --long scattered \
       --long brick \
       --long zpinch \
       --long weight: \
       -n 'decomp' -- "$@")
fi

if [ $? != 0 ] ; then usage ; exit 1 ; fi

# Note the quotes around `$TEMP': they are essential!
eval set -- "$TEMP"
while true ; do
    case "$1" in
    (-h|-H|--help)
        usage_new ; shift ;;
    (-v|--verbose)
        set -x ; shift ;;
    (-V|--viz_mesh)
        do_viz="-y" ; shift ;;
    (-d|--debug)
        debug_level="$2" ; shift 2 ;;
    (-n|--nem_slice_flag)
        nem_slice_flag="$2" ; shift 2 ;;
    (-N|--nodal)
        decomp_type="-n" ; shift ;;
    (-E|--elemental)
        decomp_type="-e" ; shift ;;
    (-i|--input)
        input=$2 ; shift 2 ;;
    (-w|--weight)
        weighting="-w $2" ; shift 2 ;;
    (-p|-j|--processors)
        processors=$2 ; shift 2 ;;
    (-R|--rootdir)
        rootdir=$2 ; shift 2 ;;
    (-S|--subdir)
        subdir=$2 ; shift 2 ;;
    (-X|--spread_suffix)
        suffix_spread=$2 ; shift 2 ;;
    (-s|--spheres_linear)
        spheres="" ; shift ;;
    (-6|--64)
        use64="-64" ; shift ;;
    (--noop)
        NOOP="--noop" ; shift ;;
    (-l)
        decomp_method="-l $2" ; shift 2 ;;

# Decomposition Method options...
    (--multikl)   decomp_method="-l multikl" ; shift ;;
    (--rcb)       decomp_method="-l rcb" ; shift ;;
    (--rcb_ignore_z)       decomp_method="-l rcb,ignore_z" ; shift ;;
    (--rib)       decomp_method="-l rib" ; shift ;;
    (--hsfc)      decomp_method="-l hsfc" ; shift ;;
    (--spectral)  decomp_method="-l spectral" ; shift ;;
    (--inertial)  decomp_method="-l inertial" ; shift ;;
    (--linear)    decomp_method="-l linear" ; shift ;;
    (--random)    decomp_method="-l random" ; shift ;;
    (--scattered) decomp_method="-l scattered" ; shift ;;
    (--brick)     decomp_method="-l brick" ; shift ;;
    (--zpinch)    decomp_method="-l zpinch" ; shift ;;
    (--) shift ; break ;;
    (*) echo "${txtred}ERROR: unrecognized option $1${txtrst}" ; shift ;;
    esac
done

########################################################################
if [ $# -eq 0 ] ; then
    echo ${txtred}
    echo "ERROR:******************************************************************"
    echo "ERROR:"
    echo "ERROR:  No 'meshfile' specified."
    echo "ERROR:"
    echo "ERROR:******************************************************************"
    echo ${txtrst}
    usage
else
    if [ -e "$1" ]
    then
        file=$1
        suffix_mesh=${file##*.}
        if test -z "$suffix_spread"
        then
            suffix_spread=$suffix_mesh
        fi
        basename=${file%.*}
    else
    echo ${txtred}
        echo "ERROR:******************************************************************"
        echo "ERROR:"
        echo "ERROR: $1 does not exist."
        echo "ERROR:"
        echo "ERROR:******************************************************************"
        echo ${txtrst}
        usage
    fi
fi

if [ -n "$input" ]
then
   if [ ! -e "$input" ]
   then
    echo ${txtred}
        echo "ERROR:******************************************************************"
        echo "ERROR:"
        echo "ERROR: Input file $input does not exist."
        echo "ERROR:"
        echo "ERROR:******************************************************************"
        echo ${txtrst}
        usage
   fi
fi

if test -z "$processors"
then
    echo
    echo ${txtred}
    echo "ERROR:***************************************************"
    echo "ERROR:"
    echo "ERROR: Must specify number of processors --processors (-p -j) option"
    echo "ERROR:"
    echo "ERROR:***************************************************"
    echo ${txtrst}
    echo
    usage
fi

if [ $processors -eq 1 ]
then
    echo ${txtgrn}
    echo "INFO: No decomposition needed for --processors = 1"
    echo ${txtrst}
    echo
    exit 0
fi

########################################################################
execute_loadbalance "$basename" "$processors" "$decomp_method" "$nem_slice_flag" "$NEM_SLICE" "$input" "$decomp_type" "$weighting"
if [ $? -ne 0 ]
then
    echo ${txtred}
    echo "ERROR:******************************************************************"
    echo "ERROR:"
    echo "ERROR     During nem_slice execution. Check error output above and rerun"
    echo "ERROR:"
    echo "ERROR:******************************************************************"
    echo ${txtrst}
    exit 1
else
    echo "${txtgrn}...nem_slice successful execution${txtrst}"
fi

########################################################################
execute_spread "$basename" "$rootdir" "$subdir" "$processors" "$NEM_SPREAD"
if [ $? -ne 0 ]
then
    echo ${txtred}
    echo "ERROR:******************************************************************"
    echo "ERROR:"
    echo "ERROR    During nem_spread execution. Check error output above and rerun"
    echo "ERROR:"
    echo "ERROR:******************************************************************"
    echo ${txtrst}
    exit 1
else
    echo "${txtgrn}...nem_spread successful execution${txtrst}"
fi

if [ ! -z $SEACAS_NO_LOGGING ]
then
    echo "SEACAS Audit logging disabled via SEACAS_NO_LOGGING setting"
else
# Echo audit information to ${ACCESS}/etc/audit.log
# Hard-wire time until get a csh-type time output ...
    time_used="0.000u 0.000s 0:00.00 0.0% 0+0k 0+0io 0pf+0w "

# Get name of user running code
    userid=$LOGNAME

# Write info to audit.log file
    if [ -w ${ACCESS}/etc/audit.log ]
    then
       date=$(date '+%m/%d/%y')
       echo "decomp ${userid} ${date} ${time_used} $(hostname)" >> ${ACCESS}/etc/audit.log
    fi
fi

echo "${txtgrn}DECOMP Successful Execution${txtrst}"