lib
/
EXODUS
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Cloned SEACAS for EXODUS library with extra build files for internal package management.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
2 Commits
1 Branch
0 Tags
19 MiB
 
 
 
 
 
 
Tag: Branch: Tree: b1e2a11951
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from 'b1e2a11951'
${ noResults }
EXODUS/packages/seacas/applications/zellij/Grid.C

1314 lines
48 KiB
Raw Blame History

// Copyright(C) 2021, 2022, 2023 National Technology & Engineering Solutions
// of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with
// NTESS, the U.S. Government retains certain rights in this software.
//
// See packages/seacas/LICENSE for details
#include <cstdlib>
#include <numeric>
#include "Cell.h"
#include "Decompose.h"
#include "Grid.h"
#include "ZE_SystemInterface.h"
#include "ZE_Version.h"
#include <Ioss_CommSet.h>
#include <Ioss_ElementBlock.h>
#include <Ioss_IOFactory.h>
#include <Ioss_NodeBlock.h>
#include <Ioss_ParallelUtils.h>
#include <Ioss_Region.h>
#include <Ioss_SideBlock.h>
#include <Ioss_SideSet.h>
#include <Ioss_SmartAssert.h>
#include <exodusII.h>
#include <fmt/chrono.h>
#include <fmt/color.h>
#include <fmt/ostream.h>
#include <open_file_limit.h>
#include <time_stamp.h>
#include <tokenize.h>
//! \file
extern unsigned int debug_level;
#if __cplusplus == 201103L
// From: https://stackoverflow.com/questions/17902405/how-to-implement-make-unique-function-in-c11
namespace std {
template <class T> struct _Unique_if
{
typedef unique_ptr<T> _Single_object;
};
template <class T> struct _Unique_if<T[]>
{
typedef unique_ptr<T[]> _Unknown_bound;
};
template <class T, size_t N> struct _Unique_if<T[N]>
{
typedef void _Known_bound;
};
template <class T, class... Args>
typename _Unique_if<T>::_Single_object make_unique(Args &&...args)
{
return unique_ptr<T>(new T(std::forward<Args>(args)...));
}
template <class T> typename _Unique_if<T>::_Unknown_bound make_unique(size_t n)
{
typedef typename remove_extent<T>::type U;
return unique_ptr<T>(new U[n]());
}
template <class T, class... Args>
typename _Unique_if<T>::_Known_bound make_unique(Args &&...) = delete;
} // namespace std
#endif
namespace {
std::string tsFormat = "[{:%H:%M:%S}]";
int axis_index(const std::string &axis_str)
{
char axis = axis_str[0];
if (axis == 'x' || axis == 'i') {
return 0;
}
if (axis == 'X' || axis == 'I') {
return 1;
}
if (axis == 'y' || axis == 'j') {
return 2;
}
if (axis == 'Y' || axis == 'J') {
return 3;
}
if (axis == 'z' || axis == 'k') {
return 4;
}
if (axis == 'Z' || axis == 'K') {
return 5;
}
return -1;
}
unsigned int which_sidesets(const std::string &sideset_string)
{
unsigned generate = 0;
for (const auto &c : sideset_string) {
if (c == 'x' || c == 'i') {
generate |= (unsigned)Flg::MIN_I;
}
if (c == 'y' || c == 'j') {
generate |= (unsigned)Flg::MIN_J;
}
if (c == 'z' || c == 'k') {
generate |= (unsigned)Flg::MIN_K;
}
if (c == 'X' || c == 'I') {
generate |= (unsigned)Flg::MAX_I;
}
if (c == 'Y' || c == 'J') {
generate |= (unsigned)Flg::MAX_J;
}
if (c == 'Z' || c == 'K') {
generate |= (unsigned)Flg::MAX_K;
}
}
return generate;
}
Ioss::PropertyManager parse_properties(SystemInterface &interFace, int int_size);
std::shared_ptr<Ioss::Region> create_input_region(const std::string &key, std::string filename,
bool ints_32_bits);
template <typename INT>
std::vector<INT> generate_node_map(Grid &grid, const Cell &cell, Mode mode, INT /*dummy*/);
void output_summary(Ioss::Region *region, std::ostream &strm);
size_t handle_elements(Grid &grid, int start_rank, int rank_count);
size_t handle_nodes(Grid &grid, int start_rank, int rank_count);
void handle_communications(Grid &grid, int start_rank, int rank_count);
void handle_surfaces(Grid &grid, int start_rank, int rank_count);
void generate_surfaces(Grid &grid, int start_rank, int rank_count);
} // namespace
Grid::Grid(SystemInterface &interFace)
: m_offset(interFace.offset()), m_scaleFactor(interFace.scale_factor()),
m_parallelSize(interFace.ranks()), m_rankCount(interFace.rank_count()),
m_startRank(interFace.start_rank()), m_equivalenceNodes(interFace.equivalence_nodes()),
m_useInternalSidesets(!interFace.ignore_internal_sidesets()),
m_subCycle(interFace.subcycle()), m_minimizeOpenFiles(interFace.minimize_open_files()),
m_generatedSideSets(which_sidesets(interFace.sideset_surfaces()))
{
set_sideset_names(interFace.sideset_names());
}
void Grid::set_extent(size_t extent_i, size_t extent_j, size_t /* unused */)
{
m_gridI = extent_i;
m_gridJ = extent_j;
m_grid.resize(m_gridI * m_gridJ);
}
bool Grid::initialize(size_t i, size_t j, const std::string &key)
{
if (unit_cells().find(key) == unit_cells().end()) {
return false;
}
const auto &unit_cell = unit_cells()[key];
SMART_ASSERT(unit_cell->m_region != nullptr)(i)(j)(key);
auto &cell = get_cell(i, j);
cell.initialize(i, j, unit_cell);
return true;
}
void Grid::add_unit_cell(const std::string &key, const std::string &unit_filename, bool ints32bit)
{
static size_t open_files = open_file_limit();
if (!minimize_open_files(Minimize::UNIT) && unit_cells().size() >= open_files) {
// Just hit the limit... Close all previous unit_cell files and set the minimize_open_files
// behavior to UNIT.
for (const auto &unit_cell : m_unitCells) {
unit_cell.second->m_region->get_database()->closeDatabase();
}
fmt::print(stderr, fmt::fg(fmt::color::yellow),
"\nWARNING: Number of unit cells exceeds open file limit. Setting "
"'mimimize_open_file' mode to UNIT.\n\n");
m_minimizeOpenFiles = Minimize((unsigned)m_minimizeOpenFiles | (unsigned)Minimize::UNIT);
}
if (unit_cells().find(key) != unit_cells().end()) {
fmt::print(stderr, fmt::fg(fmt::color::red),
"\nERROR: There is a duplicate `unit cell` ({}) in the lattice dictionary.\n\n",
key);
exit(EXIT_FAILURE);
}
auto region = create_input_region(key, unit_filename, ints32bit);
if (region == nullptr) {
fmt::print(stderr, fmt::fg(fmt::color::red),
"\nERROR: Unable to open the database '{}' associated with the unit cell '{}'.\n\n",
unit_filename, key);
exit(EXIT_FAILURE);
}
unit_cells().emplace(key, std::make_shared<UnitCell>(region));
if (minimize_open_files(Minimize::UNIT)) {
unit_cells()[key]->m_region->get_database()->closeDatabase();
}
if (debug_level & 2) {
m_pu.progress(fmt::format("\tCreated Unit Cell {}", key));
}
}
void Grid::set_coordinate_offsets()
{
// All unit_cells have same X, Y (and Z) extent. Only need X and Y
auto x_range = get_cell(0, 0).get_coordinate_range(Axis::X);
auto y_range = get_cell(0, 0).get_coordinate_range(Axis::Y);
auto delta_x = x_range.second - x_range.first;
auto delta_y = y_range.second - y_range.first;
for (size_t j = 0; j < JJ(); j++) {
for (size_t i = 0; i < II(); i++) {
auto &cell = get_cell(i, j);
cell.m_offX = delta_x * (double)i;
cell.m_offY = delta_y * (double)j;
if (debug_level & 2) {
util().progress(
fmt::format("\tCell({}, {}) X = {}, Y = {}", i, j, cell.m_offX, cell.m_offY));
}
}
}
}
void Grid::create_output_regions(SystemInterface &interFace)
{
if (debug_level & 2) {
util().progress(__func__);
}
m_outputRegions.resize(interFace.ranks());
int int_size = interFace.ints32bit() ? 4 : 8;
Ioss::PropertyManager properties = parse_properties(interFace, int_size);
if (parallel_size() == 1) {
properties.add(Ioss::Property("OMIT_EXODUS_NUM_MAPS", 1));
}
// Disable this for now. Readers need to be modified and propagated to allow this.
// properties.add(Ioss::Property("MINIMAL_NEMESIS_DATA", 1));
if (debug_level & 2) {
properties.add(Ioss::Property("ENABLE_TRACING", 1));
}
for (int i = m_startRank; i < m_startRank + m_rankCount; i++) {
// Define the output database(s)...
if (m_parallelSize > 1) {
properties.add(Ioss::Property("processor_count", parallel_size()));
properties.add(Ioss::Property("my_processor", i));
}
Ioss::DatabaseIO *dbo =
Ioss::IOFactory::create("exodus", interFace.outputName_, Ioss::WRITE_RESTART,
Ioss::ParallelUtils::comm_self(), properties);
if (dbo == nullptr || !dbo->ok(true)) {
std::exit(EXIT_FAILURE);
}
m_outputRegions[i] = std::make_unique<Ioss::Region>(dbo, "zellij_output_region");
output_region(i)->begin_mode(Ioss::STATE_DEFINE_MODEL);
output_region(i)->property_add(Ioss::Property("code_name", qainfo[0]));
output_region(i)->property_add(Ioss::Property("code_version", qainfo[2]));
if (interFace.minimize_open_files() == Minimize::OUTPUT ||
interFace.minimize_open_files() == Minimize::ALL) {
output_region(i)->get_database()->closeDatabase();
}
}
}
void Grid::set_sideset_names(const std::string &names)
{
// 'names' is a string of the form "Axis:name,Axis:name, ..."
// For example "x:left,y:top,X:right,Y:bottom"
// Parse the list and only update the names specified in the list...
if (names.empty()) {
return;
}
auto tokens = Ioss::tokenize(names, ",");
for (auto &token : tokens) {
auto axis = token.substr(0, 1);
bool valid = axis.find_first_not_of("ijkIJKxyzXYZ") == std::string::npos;
if (!valid) {
fmt::print(stderr, fmt::fg(fmt::color::red),
"\nERROR: Invalid axis '{}' specified for sideset name. Valid is one of "
"'ijkIJKxyzXYZ'.\n\n",
axis);
exit(EXIT_FAILURE);
}
// Now get the name...
auto ss_name = token.substr(2);
// Update the name in the list of generated sideset names...
auto index = axis_index(axis);
SMART_ASSERT(index >= 0)(axis)(index);
generated_surface_names[index] = std::move(ss_name);
}
}
void Grid::decompose(const std::string &method)
{
if (debug_level & 2) {
util().progress(__func__);
}
decompose_grid(*this, m_parallelSize, method);
categorize_processor_boundaries();
}
void Grid::categorize_processor_boundaries()
{
if (debug_level & 2) {
util().progress(__func__);
}
// Now iterate the cells and tell each cell the rank of all neighboring cells...
// boundary with its "left" or "lower" neighboring cell.
for (size_t j = 0; j < JJ(); j++) {
for (size_t i = 0; i < II(); i++) {
auto &cell = get_cell(i, j);
if (i > 0) {
const auto &left = get_cell(i - 1, j);
cell.set_rank(Loc::L, left.rank(Loc::C));
if (j > 0) {
const auto &BL = get_cell(i - 1, j - 1);
cell.set_rank(Loc::BL, BL.rank(Loc::C));
}
if (j < JJ() - 1) {
const auto &TL = get_cell(i - 1, j + 1);
cell.set_rank(Loc::TL, TL.rank(Loc::C));
}
}
if (i < II() - 1) {
const auto &right = get_cell(i + 1, j);
cell.set_rank(Loc::R, right.rank(Loc::C));
if (j > 0) {
const auto &BR = get_cell(i + 1, j - 1);
cell.set_rank(Loc::BR, BR.rank(Loc::C));
}
if (j < JJ() - 1) {
const auto &TR = get_cell(i + 1, j + 1);
cell.set_rank(Loc::TR, TR.rank(Loc::C));
}
}
if (j > 0) {
const auto &B = get_cell(i, j - 1);
cell.set_rank(Loc::B, B.rank(Loc::C));
}
if (j < JJ() - 1) {
const auto &T = get_cell(i, j + 1);
cell.set_rank(Loc::T, T.rank(Loc::C));
}
}
}
if (debug_level & 32) {
auto width = Ioss::Utils::number_width(parallel_size());
for (size_t j = 0; j < JJ(); j++) {
for (size_t i = 0; i < II(); i++) {
const auto &cell = get_cell(i, j);
auto left = cell.processor_boundary(Loc::L) ? '<' : ' ';
auto below = cell.processor_boundary(Loc::B) ? '^' : ' ';
fmt::print(" {0}{1:{3}}{2}", left, cell.rank(Loc::C), below, width);
}
fmt::print("\n");
}
}
}
void Grid::generate_sidesets()
{
if (m_generatedSideSets != 0) {
for (const auto &unit_cell : m_unitCells) {
unit_cell.second->generate_boundary_faces(m_generatedSideSets);
}
}
}
void Grid::internal_process()
{
if (debug_level & 2) {
util().progress(__func__);
}
auto node_count = handle_nodes(*this, m_startRank, m_rankCount);
auto element_count = handle_elements(*this, m_startRank, m_rankCount);
handle_communications(*this, m_startRank, m_rankCount);
if (m_useInternalSidesets) {
handle_surfaces(*this, m_startRank, m_rankCount);
}
generate_surfaces(*this, m_startRank, m_rankCount);
for (int i = m_startRank; i < m_startRank + m_rankCount; i++) {
output_region(i)->end_mode(Ioss::STATE_DEFINE_MODEL);
if (debug_level & 64) {
output_summary(output_region(i), std::cerr);
}
}
if (util().parallel_rank() == 0) {
fmt::print(" {} Nodes; {} Elements.\n", fmt::group_digits(node_count),
fmt::group_digits(element_count));
}
}
template void Grid::process(SystemInterface &, int64_t);
template void Grid::process(SystemInterface &, int);
template <typename INT> void Grid::process(SystemInterface &interFace, INT /* dummy */)
{
bool subcycle = m_subCycle;
int start_rank = m_startRank;
int rank_count = m_rankCount;
int end_rank = subcycle ? m_parallelSize : m_startRank + m_rankCount;
if (end_rank > m_parallelSize) {
end_rank = m_parallelSize;
m_rankCount = m_parallelSize - m_startRank;
}
for (int begin = start_rank; begin < end_rank; begin += rank_count) {
m_startRank = begin;
if (m_startRank + m_rankCount > m_parallelSize) {
m_rankCount = m_parallelSize - m_startRank;
}
if (debug_level & 2) {
fmt::print(stderr, "{} Processing Ranks {} to {}\n", time_stamp(tsFormat), begin,
begin + rank_count - 1);
}
create_output_regions(interFace);
internal_process();
if (debug_level & 2) {
fmt::print(stderr, "{} Lattice Processing Finalized\n", time_stamp(tsFormat));
}
output_model(INT(0));
if (debug_level & 2) {
fmt::print(stderr, "{} Model Output\n", time_stamp(tsFormat));
}
}
}
template void Grid::output_model(int64_t);
template void Grid::output_model(int);
// Write the output database(s) for all ranks...
template <typename INT> void Grid::output_model(INT /*dummy*/)
{
if (debug_level & 2) {
util().progress(__func__);
}
// Coordinates do not depend on order of operations, so can do these
// a rank at a time which can be more efficient once we exceed
// maximum number of open files... Also easier to parallelize...
for (int r = m_startRank; r < m_startRank + m_rankCount; r++) {
for (size_t j = 0; j < JJ(); j++) {
for (size_t i = 0; i < II(); i++) {
const auto &cell = get_cell(i, j);
if (cell.rank(Loc::C) == r) {
output_nodal_coordinates(cell);
}
}
}
if (minimize_open_files(Minimize::OUTPUT)) {
output_region(r)->get_database()->closeDatabase();
}
}
if (debug_level & 2) {
util().progress("\tEnd Nodal Coordinate Output");
}
// Surfaces do not depend on order of operations, so can do these
// a rank at a time which can be more efficient once we exceed
// maximum number of open files... Also easier to parallelize...
for (int r = m_startRank; r < m_startRank + m_rankCount; r++) {
for (size_t j = 0; j < JJ(); j++) {
for (size_t i = 0; i < II(); i++) {
auto &cell = get_cell(i, j);
if (cell.rank(Loc::C) == r) {
if (m_useInternalSidesets) {
output_surfaces(cell, INT(0));
}
output_generated_surfaces(cell, INT(0));
}
}
}
if (minimize_open_files(Minimize::OUTPUT)) {
output_region(r)->get_database()->closeDatabase();
}
}
if (debug_level & 2) {
util().progress("\tEnd Surface Output");
}
// All the rest of these depend on progressing through the cells in
// correct order so that can pass information correctly from cell to
// cell. Need to figure out how to eliminate this ordering so can
// parallelize and not worry about open file count.
for (size_t j = 0; j < JJ(); j++) {
for (size_t i = 0; i < II(); i++) {
auto &cell = get_cell(i, j);
auto node_map = generate_node_map(*this, cell, Mode::PROCESSOR, INT(0));
output_block_connectivity(cell, node_map);
if (parallel_size() > 1) {
output_nodal_communication_map(cell, node_map);
}
}
}
if (debug_level & 2) {
util().progress("\tEnd Nodal Communication Map Output");
}
if (parallel_size() > 1) {
for (size_t j = 0; j < JJ(); j++) {
for (size_t i = 0; i < II(); i++) {
auto &cell = get_cell(i, j);
output_node_map(cell, INT(0));
output_element_map(cell, INT(0));
}
}
if (debug_level & 2) {
util().progress("\tEnd Node/Element Map Output");
}
}
}
void Grid::output_nodal_coordinates(const Cell &cell)
{
int rank = cell.rank(Loc::C);
auto *nb = cell.region()->get_node_blocks()[0];
std::vector<double> coord_x;
std::vector<double> coord_y;
std::vector<double> coord_z;
// Are we modifying the coordinates ... scale and/or offset and/or offset_unit_cell...
bool mod_x = cell.m_offX != 0.0 || m_scaleFactor != 1.0 || m_offset[0] != 0.0;
bool mod_y = cell.m_offY != 0.0 || m_scaleFactor != 1.0 || m_offset[1] != 0.0;
bool mod_z = m_scaleFactor != 1.0 || m_offset[2] != 0.0;
double scale = m_scaleFactor;
nb->get_field_data("mesh_model_coordinates_x", coord_x);
if (mod_x) {
double offset = m_offset[0];
std::for_each(coord_x.begin(), coord_x.end(),
[&cell, scale, offset](double &d) { d = (d + cell.m_offX) * scale + offset; });
}
nb->get_field_data("mesh_model_coordinates_y", coord_y);
if (mod_y) {
double offset = m_offset[1];
std::for_each(coord_y.begin(), coord_y.end(),
[&cell, scale, offset](double &d) { d = (d + cell.m_offY) * scale + offset; });
}
nb->get_field_data("mesh_model_coordinates_z", coord_z);
if (mod_z) {
double offset = m_offset[2];
std::for_each(coord_z.begin(), coord_z.end(),
[scale, offset](double &d) { d = d * scale + offset; });
}
// Filter coordinates down to only "new nodes"...
if (m_equivalenceNodes && (cell.has_neighbor_i() || cell.has_neighbor_j())) {
auto mode = parallel_size() > 1 ? Mode::PROCESSOR : Mode::GLOBAL;
auto categorized_nodes = cell.categorize_nodes(mode);
size_t nn = 0;
for (size_t n = 0; n < categorized_nodes.size(); n++) {
if (categorized_nodes[n] == 0) {
coord_x[nn] = coord_x[n];
coord_y[nn] = coord_y[n];
coord_z[nn] = coord_z[n];
nn++;
}
}
}
int exoid = output_region(rank)->get_database()->get_file_pointer();
auto start = cell.m_localNodeIdOffset + 1;
auto count = cell.added_node_count(Mode::PROCESSOR, m_equivalenceNodes);
ex_put_partial_coord(exoid, start, count, coord_x.data(), coord_y.data(), coord_z.data());
if (minimize_open_files(Minimize::UNIT)) {
cell.region()->get_database()->closeDatabase();
}
}
template <typename INT> void Grid::output_generated_surfaces(Cell &cell, INT /*dummy*/)
{
auto generated_ijk = get_generated_sidesets();
if (generated_ijk == 0) {
return;
}
int rank = cell.rank(Loc::C);
const std::array<int, 6> boundary_rank{
cell.rank(Loc::L), cell.rank(Loc::R), cell.rank(Loc::B), cell.rank(Loc::T), -1, -1};
std::array<enum Flg, 6> boundary_flag{Flg::MIN_I, Flg::MAX_I, Flg::MIN_J,
Flg::MAX_J, Flg::MIN_K, Flg::MAX_K};
int exoid = output_region(rank)->get_database()->get_file_pointer();
for (int face = 0; face < 6; face++) {
if ((generated_ijk & (unsigned)boundary_flag[face]) && boundary_rank[face] == -1) {
// Find surface on output mesh...
auto *osurf = output_region(rank)->get_sideset(generated_surface_names[face]);
SMART_ASSERT(osurf != nullptr);
auto &oblocks = osurf->get_side_blocks();
SMART_ASSERT(oblocks.size() == 1)(oblocks.size());
auto &boundary = cell.unit()->boundary_blocks[face];
auto count = boundary.size();
std::vector<INT> elements;
std::vector<INT> faces;
elements.reserve(count);
faces.reserve(count);
for (const auto &block_faces : boundary.m_faces) {
const auto &block_name = block_faces.first;
const auto &bnd_faces = block_faces.second;
// This is the offset within this element block -- i.e., the 'element_offsetth' element in
// this block.
size_t element_offset = cell.m_localElementIdOffset[block_name];
// This is the offset of the elements in this element block within all of the elements in
// the output file.
size_t global_offset = output_region(rank)->get_element_block(block_name)->get_offset();
for (const auto &bface : bnd_faces) {
elements.push_back(bface / 10 + element_offset + global_offset);
faces.push_back(bface % 10 + 1);
}
}
auto id = osurf->get_property("id").get_int();
auto start = cell.m_localSurfaceOffset[generated_surface_names[face]];
ex_put_partial_set(exoid, EX_SIDE_SET, id, start + 1, count, elements.data(), faces.data());
}
}
}
template <typename INT> void Grid::output_surfaces(Cell &cell, INT /*dummy*/)
{
int rank = cell.rank(Loc::C);
int exoid = output_region(rank)->get_database()->get_file_pointer();
// Get the surfaces on this cell...
const auto &surfaces = cell.region()->get_sidesets();
for (const auto *surface : surfaces) {
// Find corresponding surface on output mesh...
auto *osurf = output_region(rank)->get_sideset(surface->name());
SMART_ASSERT(osurf != nullptr);
const auto &oblocks = osurf->get_side_blocks();
std::vector<INT> elements;
std::vector<INT> faces;
elements.reserve(oblocks[0]->entity_count());
faces.reserve(oblocks[0]->entity_count());
const auto &blocks = surface->get_side_blocks();
for (const auto *block : blocks) {
// Get the element/face pairs for the SideBlock in this surface...
std::vector<INT> element_side;
block->get_field_data("element_side_raw", element_side);
// Now separate the element and the face into different vectors and
// update the element number to match the elements in this cell...
const auto *parent = block->parent_element_block();
SMART_ASSERT(parent != nullptr);
// This is the offset of the elements in the unit cell parent element block
size_t unit_block_offset = parent->get_offset();
// This is the offset within this element block -- i.e., the 'element_offsetth' element in
// this block.
size_t element_offset = cell.m_localElementIdOffset[parent->name()];
// This is the offset of the elements in this element block within all of the elements in the
// output file.
size_t global_offset = output_region(rank)->get_element_block(parent->name())->get_offset();
size_t entity_count = block->entity_count();
for (size_t i = 0; i < entity_count; i++) {
auto element = element_side[2 * i + 0];
auto output_element = element - unit_block_offset + element_offset + global_offset;
elements.push_back(output_element);
faces.push_back(element_side[2 * i + 1]);
}
}
auto id = osurf->get_property("id").get_int();
auto start = cell.m_localSurfaceOffset[osurf->name()];
auto count = elements.size();
ex_put_partial_set(exoid, EX_SIDE_SET, id, start + 1, count, elements.data(), faces.data());
}
if (minimize_open_files(Minimize::UNIT)) {
cell.region()->get_database()->closeDatabase();
}
}
template <typename INT>
void Grid::output_block_connectivity(Cell &cell, const std::vector<INT> &node_map)
{
int rank = cell.rank(Loc::C);
if (rank >= m_startRank && rank < m_startRank + m_rankCount) {
int exoid = output_region(rank)->get_database()->get_file_pointer();
const auto &blocks = cell.region()->get_element_blocks();
std::vector<INT> connect;
for (const auto *block : blocks) {
block->get_field_data("connectivity_raw", connect);
for (size_t k = 0; k < connect.size(); k++) {
connect[k] = node_map[connect[k]];
}
auto start = cell.m_localElementIdOffset[block->name()] + 1;
auto count = block->entity_count();
auto id = block->get_property("id").get_int();
if (debug_level & 8) {
fmt::print(stderr, "Rank: {}, Cell({}, {}), Block {}, id {}, start {}, count {}\n", rank,
cell.m_i, cell.m_j, block->name(), id, start, count);
}
ex_put_partial_conn(exoid, EX_ELEM_BLOCK, id, start, count, connect.data(), nullptr, nullptr);
}
if (debug_level & 2) {
util().progress(fmt::format("Generated Node Map / Output Connectivity for Cell({}, {})",
cell.m_i, cell.m_j));
}
if (minimize_open_files(Minimize::UNIT)) {
cell.region()->get_database()->closeDatabase();
}
if (minimize_open_files(Minimize::OUTPUT)) {
output_region(rank)->get_database()->closeDatabase();
}
}
}
template <typename INT>
void Grid::output_nodal_communication_map(Cell &cell, const std::vector<INT> &node_map)
{
// The `node_map` has the processor-local node ids for all nodes on this cell.
// Note that the `node_map` starts at index 1 in the vector...
// Need to check the boundaries of the cell and determine which boundaries are on a different rank
// and output the nodes and processors to the communication map.
int rank = cell.rank(Loc::C);
if (rank >= m_startRank && rank < m_startRank + m_rankCount) {
std::vector<INT> nodes;
std::vector<INT> procs;
cell.populate_node_communication_map(node_map, nodes, procs);
int exoid = output_region(rank)->get_database()->get_file_pointer();
auto start = cell.m_communicationNodeOffset + 1;
auto count = cell.m_communicationNodeCount;
ex_put_partial_node_cmap(exoid, 1, start, count, nodes.data(), procs.data(), rank);
if (minimize_open_files(Minimize::OUTPUT)) {
output_region(rank)->get_database()->closeDatabase();
}
if (debug_level & 32) {
fmt::print(stderr, "Rank: {}, Cell({}, {}), Node Comm Map: start {}, count {}\n", rank,
cell.m_i, cell.m_j, start, count);
}
if (debug_level & 2) {
util().progress(
fmt::format("Output Nodal Communication Map for Cell({}, {})", cell.m_i, cell.m_j));
}
}
}
template <typename INT> void Grid::output_node_map(const Cell &cell, INT /*dummy*/)
{
int rank = cell.rank(Loc::C);
auto start = cell.m_localNodeIdOffset + 1;
auto count = cell.added_node_count(Mode::PROCESSOR, m_equivalenceNodes);
if (parallel_size() == 1) {
auto gid = cell.m_globalNodeIdOffset + 1;
std::vector<INT> map(count);
std::iota(map.begin(), map.end(), gid);
int exoid = output_region(rank)->get_database()->get_file_pointer();
ex_put_partial_id_map(exoid, EX_NODE_MAP, start, count, map.data());
}
else {
auto map = generate_node_map(*this, cell, Mode::GLOBAL, INT(0));
if (rank >= m_startRank && rank < m_startRank + m_rankCount) {
// Filter nodes down to only "new nodes"...
if (m_equivalenceNodes && (cell.has_neighbor_i() || cell.has_neighbor_j())) {
auto mode = Mode::PROCESSOR;
auto categorized_nodes = cell.categorize_nodes(mode);
size_t nn = 0;
for (size_t n = 0; n < categorized_nodes.size(); n++) {
if (categorized_nodes[n] == 0) {
map[nn + 1] = map[n + 1];
nn++;
}
}
}
if (debug_level & 8) {
fmt::print("Cell({}, {}), start {}, count {}\n", cell.m_i, cell.m_j, start, count);
}
int exoid = output_region(rank)->get_database()->get_file_pointer();
ex_put_partial_id_map(exoid, EX_NODE_MAP, start, count, &map[1]);
if (minimize_open_files(Minimize::OUTPUT)) {
output_region(rank)->get_database()->closeDatabase();
}
}
}
if (debug_level & 2) {
util().progress(fmt::format("Generated Node Map for Rank {}, Cell({}, {}): start {}, count "
"{}\n",
rank, cell.m_i, cell.m_j, start, count));
}
}
template <typename INT> void Grid::output_element_map(Cell &cell, INT /*dummy*/)
{
int rank = cell.rank(Loc::C);
if (rank >= m_startRank && rank < m_startRank + m_rankCount) {
int exoid = output_region(rank)->get_database()->get_file_pointer();
const auto &output_blocks = output_region(rank)->get_element_blocks();
// This is the element block offset for the "single output file"
// for the block being output For example, if the total mesh has
// 3 blocks, with 100, 200, 100 elements, then the element block
// offset would be 0, 100, 300 (Ioss::ElementBlock::get_offset())
size_t global_id_offset = 0;
for (const auto *output_element_block : output_blocks) {
auto *block = cell.region()->get_element_block(output_element_block->name());
if (block != nullptr) {
auto gid = cell.m_globalElementIdOffset[block->name()] + 1 + global_id_offset;
auto element_count = block->entity_count();
std::vector<INT> map(element_count);
std::iota(map.begin(), map.end(), gid);
auto output_block_offset = output_element_block->get_offset();
// This cells element block ids start this far into the portion of the map for this
// element block
auto local_offset = cell.m_localElementIdOffset[block->name()];
auto start = output_block_offset + local_offset + 1;
ex_put_partial_id_map(exoid, EX_ELEM_MAP, start, element_count, map.data());
if (debug_level & 8) {
fmt::print("Rank {}: Cell({}, {}), Block {}, start {}, element_count {}, gid {}\n", rank,
cell.m_i, cell.m_j, block->name(), start, element_count, gid);
}
}
// If we were outputting a single file, then this element
// block in that file would have this many elements.
auto global_block_element_count =
output_element_block->get_property("global_entity_count").get_int();
global_id_offset += global_block_element_count;
}
if (minimize_open_files(Minimize::OUTPUT)) {
output_region(rank)->get_database()->closeDatabase();
}
}
}
void Grid::handle_file_count()
{
// If the user has specified a 'minimize_open_files' behavior on the command line,
// honor that mode here. Note that we have already processed the unit cells and
// if the unit cell count exceeds the open file limit, then that code has already
// automatically set the minimize_open_files to include UNIT...
if (m_minimizeOpenFiles == Minimize::ALL) {
return;
}
size_t open_files = open_file_limit();
if (util().parallel_rank() == 0) {
fmt::print("\n Maximum Open File Count = {}\n", open_file_limit());
}
auto unit_cell_size = unit_cells().size();
if (minimize_open_files(Minimize::UNIT)) {
unit_cell_size = 1; // There will only be a single unit cell file open at a time.
}
if (unit_cell_size + m_rankCount > open_files) {
// Too many files, need to close some of them after accessing...
// We have already checked the unit_cell() size earlier.
// If user has specified 'minimize_open_files == OUTPUT', then honor that...
if (minimize_open_files(Minimize::OUTPUT)) {
return;
}
// Have decision on whether to keep all unit_files open
// and reduce open output files further, or to close
// unit files and have more output files open...
int output_open = open_files - (int)unit_cell_size;
if (output_open < (int)(.2 * m_rankCount)) {
// Close the unit files. Prefer to have output files open instead.
m_minimizeOpenFiles = Minimize((unsigned)m_minimizeOpenFiles | (unsigned)Minimize::UNIT);
unit_cell_size = 1;
}
// Set the rank count to the number of files left after subtracting off 'unit_cell_size'
// and set the 'subcycle' mode.
auto max_rank_count = open_files - unit_cell_size;
if (max_rank_count < (size_t)m_rankCount) {
m_rankCount = max_rank_count;
}
m_subCycle = true;
}
if (util().parallel_rank() == 0 && m_minimizeOpenFiles != Minimize::NONE) {
std::array<std::string, 4> smode{"NONE", "UNIT", "OUTPUT", "ALL"};
fmt::print(" Setting `minimize_open_files` mode to {}.\n", smode[(int)m_minimizeOpenFiles]);
}
}
namespace {
size_t handle_elements(Grid &grid, int start_rank, int rank_count)
{
// Not all unit cells have the same element blocks and the output
// grid will contain the union of the element blocks on each unit
// cell...
//
// While finalizing the cells, we will create this union for use in
// the output region. Would be quicker to do iteration on unit_cell
// map which is smaller, but for now do it here and see if becomes
// bottleneck.
std::map<std::string, std::unique_ptr<Ioss::ElementBlock>> output_element_blocks;
std::vector<std::map<std::string, size_t>> element_block_elem_count(grid.parallel_size());
std::map<std::string, size_t> global_element_block_elem_count;
for (size_t j = 0; j < grid.JJ(); j++) {
for (size_t i = 0; i < grid.II(); i++) {
auto &cell = grid.get_cell(i, j);
auto rank = cell.rank(Loc::C);
const auto &element_blocks = cell.region()->get_element_blocks();
for (const auto *block : element_blocks) {
auto &blk = block->name();
cell.m_globalElementIdOffset[blk] = global_element_block_elem_count[blk];
cell.m_localElementIdOffset[blk] = element_block_elem_count[rank][blk];
element_block_elem_count[rank][blk] += block->entity_count();
global_element_block_elem_count[blk] += block->entity_count();
if (debug_level & 8) {
fmt::print("rank, i, j, blk, loffset, goffset, block_count: {}: {} {} {} {} {} {}\n",
rank, i, j, blk, cell.m_localElementIdOffset[blk],
cell.m_globalElementIdOffset[blk], element_block_elem_count[rank][blk]);
}
// Create output element block if does not exist yet...
if (output_element_blocks.find(blk) == output_element_blocks.end()) {
output_element_blocks.emplace(
blk, std::make_unique<Ioss::ElementBlock>(Ioss::ElementBlock(*block)));
}
}
}
}
// Calculate values needed to set the "global_entity_count" property on the output element
// blocks.
size_t global_element_count = 0;
std::map<const std::string, int64_t> global_block_element_count;
for (auto &blk : output_element_blocks) {
for (int rank = 0; rank < grid.parallel_size(); rank++) {
auto *block = blk.second.get();
global_block_element_count[block->name()] += element_block_elem_count[rank][block->name()];
global_element_count += element_block_elem_count[rank][block->name()];
}
}
// Define the element blocks in the output database...
for (int rank = start_rank; rank < start_rank + rank_count; rank++) {
for (auto &blk : output_element_blocks) {
auto *block = new Ioss::ElementBlock(*blk.second.get());
block->property_update("entity_count", element_block_elem_count[rank][block->name()]);
block->property_update("global_entity_count", global_block_element_count[block->name()]);
grid.output_region(rank)->property_add(
Ioss::Property("global_element_count", (int64_t)global_element_count));
grid.output_region(rank)->add(block);
if (debug_level & 8) {
fmt::print("rank, blk, element_count: {}: {} {}\n", rank, block->name(),
block->entity_count());
}
}
}
return global_element_count;
}
size_t handle_nodes(Grid &grid, int start_rank, int rank_count)
{
size_t global_node_count = 0;
std::vector<size_t> local_node_count(grid.parallel_size());
for (size_t j = 0; j < grid.JJ(); j++) {
for (size_t i = 0; i < grid.II(); i++) {
auto &cell = grid.get_cell(i, j);
auto rank = cell.rank(Loc::C);
cell.m_globalNodeIdOffset = global_node_count;
cell.m_localNodeIdOffset = local_node_count[rank];
SMART_ASSERT(cell.region() != nullptr)(i)(j);
auto new_global_nodes = cell.added_node_count(Mode::GLOBAL, grid.equivalence_nodes());
auto new_processor_nodes = cell.added_node_count(Mode::PROCESSOR, grid.equivalence_nodes());
if (debug_level & 8) {
fmt::print("rank: i, j, node_offset, added_nodes: {}: {} {} {} {} {}\n", rank, i, j,
local_node_count[rank], new_global_nodes, new_processor_nodes);
}
local_node_count[rank] += new_processor_nodes;
global_node_count += new_global_nodes;
}
}
// Define the output database node block...
for (int i = start_rank; i < start_rank + rank_count; i++) {
std::string block_name = "nodeblock_1";
int spatial_dimension = 3;
auto block = new Ioss::NodeBlock(grid.output_region(i)->get_database(), block_name,
local_node_count[i], spatial_dimension);
block->property_add(Ioss::Property("id", 1));
grid.output_region(i)->add(block);
grid.output_region(i)->property_add(
Ioss::Property("global_node_count", (int64_t)global_node_count));
}
return global_node_count;
}
void handle_communications(Grid &grid, int start_rank, int rank_count)
{
// Need to determine the number of nodes on the processor
// boundaries for each output database and create a CommSet...
std::vector<size_t> bnode(grid.parallel_size());
for (size_t j = 0; j < grid.JJ(); j++) {
for (size_t i = 0; i < grid.II(); i++) {
auto &cell = grid.get_cell(i, j);
auto rank = cell.rank(Loc::C);
cell.m_communicationNodeOffset = bnode[rank];
auto bnd = cell.processor_boundary_node_count();
bnode[rank] += bnd;
if (debug_level & 32) {
fmt::print("rank: {}, Cell({}, {}): Boundary Count = {}, Total = {}\n", rank, i, j, bnd,
bnode[rank]);
}
}
}
// Now add communication sets to all output databases...
for (int rank = start_rank; rank < start_rank + rank_count; rank++) {
auto *cs = new Ioss::CommSet(grid.output_region(rank)->get_database(), "commset_node", "node",
bnode[rank]);
grid.output_region(rank)->add(cs);
}
}
void handle_surfaces(Grid &grid, int start_rank, int rank_count)
{
// Not all unit cells have the same surfaces and the output
// grid will contain the union of the surfaces on each unit
// cell that has a surface on the boundary of the output mesh.
//
// While finalizing the cells, we will create this union for use in
// the output region. Would be quicker to do iteration on unit_cell
// map which is smaller, but for now do it here and see if becomes
// bottleneck.
std::map<std::string, std::unique_ptr<Ioss::SideSet>> output_surfaces;
std::map<std::string, size_t> global_surface_face_count;
// Per rank...
std::vector<std::map<std::string, size_t>> surface_face_count(grid.parallel_size());
std::vector<std::map<std::string, size_t>> local_surface_offset(grid.parallel_size());
for (size_t j = 0; j < grid.JJ(); j++) {
for (size_t i = 0; i < grid.II(); i++) {
auto &cell = grid.get_cell(i, j);
auto rank = cell.rank(Loc::C);
const auto &surfaces = cell.region()->get_sidesets();
for (const auto *surface : surfaces) {
auto &surf = surface->name();
cell.m_localSurfaceOffset[surf] = local_surface_offset[rank][surf];
const auto &blocks = surface->get_side_blocks();
for (const auto *blk : blocks) {
surface_face_count[rank][surf] += blk->entity_count();
global_surface_face_count[surf] += blk->entity_count();
local_surface_offset[rank][surf] += blk->entity_count();
}
// Create output surface if does not exist yet...
if (output_surfaces.find(surf) == output_surfaces.end()) {
output_surfaces.emplace(surf, std::make_unique<Ioss::SideSet>(Ioss::SideSet(*surface)));
}
}
}
}
// Define the surfaces in the output database...
for (int rank = start_rank; rank < start_rank + rank_count; rank++) {
for (auto &surf : output_surfaces) {
auto *surface =
new Ioss::SideSet(grid.output_region(rank)->get_database(), surf.second->name());
auto *block =
new Ioss::SideBlock(grid.output_region(rank)->get_database(), surf.second->name(),
"quad4", "hex8", surface_face_count[rank][surface->name()]);
surface->add(block);
block->property_update("global_entity_count", global_surface_face_count[surface->name()]);
surface->property_update("global_entity_count", global_surface_face_count[surface->name()]);
block->property_update("distribution_factor_count", 0);
grid.output_region(rank)->add(surface);
}
}
}
void generate_surfaces(Grid &grid, int start_rank, int rank_count)
{
// The user may have requested that sidesets on one or more
// of the models boundary surfaces (min i,j,k and/or max i,j,k)
// be automatically generated (they don't exist on the input
// unit cells). Handle those sidesets here.
auto generated_ijk = grid.get_generated_sidesets();
if (generated_ijk == 0) {
return;
}
std::array<size_t, 6> global_surface_face_count{};
// Per rank...
std::vector<std::array<size_t, 6>> surface_face_count(grid.parallel_size());
std::vector<std::array<size_t, 6>> local_surface_offset(grid.parallel_size());
std::array<enum Flg, 6> boundary_flag{Flg::MIN_I, Flg::MAX_I, Flg::MIN_J,
Flg::MAX_J, Flg::MIN_K, Flg::MAX_K};
for (size_t j = 0; j < grid.JJ(); j++) {
for (size_t i = 0; i < grid.II(); i++) {
auto &cell = grid.get_cell(i, j);
auto rank = cell.rank(Loc::C);
const std::array<int, 6> boundary_rank{
cell.rank(Loc::L), cell.rank(Loc::R), cell.rank(Loc::B), cell.rank(Loc::T), -1, -1};
for (int face = 0; face < 6; face++) {
if ((generated_ijk & (unsigned)boundary_flag[face]) && boundary_rank[face] == -1) {
cell.m_localSurfaceOffset[grid.generated_surface_names[face]] =
local_surface_offset[rank][face];
auto count = cell.unit()->boundary_blocks[face].size();
global_surface_face_count[face] += count;
surface_face_count[rank][face] += count;
local_surface_offset[rank][face] += count;
}
}
}
}
// Define the surfaces in the output database...
for (int rank = start_rank; rank < start_rank + rank_count; rank++) {
for (size_t i = 0; i < 6; i++) {
if (global_surface_face_count[i] > 0) {
auto *surface = new Ioss::SideSet(grid.output_region(rank)->get_database(),
grid.generated_surface_names[i]);
auto *block = new Ioss::SideBlock(grid.output_region(rank)->get_database(),
grid.generated_surface_names[i], "quad4", "hex8",
surface_face_count[rank][i]);
surface->property_update("global_entity_count", global_surface_face_count[i]);
block->property_update("global_entity_count", global_surface_face_count[i]);
block->property_update("distribution_factor_count", 0);
surface->add(block);
grid.output_region(rank)->add(surface);
}
}
}
}
template <typename INT>
std::vector<INT> generate_node_map(Grid &grid, const Cell &cell, Mode mode, INT /*dummy*/)
{
// Generate a "map" from nodes in the input connectivity to the
// output connectivity in global nodes. If no neighbors, then
// this would just be adding `cell.m_globalNodeIdOffset` to each
// connectivity entry.
std::vector<INT> map = cell.generate_node_map(mode, grid.equivalence_nodes(), INT(0));
if (debug_level & 8) {
fmt::print("Cell({},{}) PROCESSOR MAP: {}\n", cell.m_i, cell.m_j, fmt::join(map, " "));
}
// Now that we have the node map for this cell, we need to save
// the mappings for the max_I and max_J faces and max_I-max_J edge
// for use by later neighbors... Check whether cell has neighbors
// on max_I or max_J faces...
if (grid.equivalence_nodes()) {
if ((mode == Mode::GLOBAL ||
(mode == Mode::PROCESSOR && cell.rank(Loc::C) == cell.rank(Loc::R))) &&
(cell.m_i + 1 < grid.II())) {
const auto &neighbor = grid.get_cell(cell.m_i + 1, cell.m_j);
cell.populate_neighbor(Loc::L, map, neighbor);
}
if ((mode == Mode::GLOBAL ||
(mode == Mode::PROCESSOR && cell.rank(Loc::C) == cell.rank(Loc::T))) &&
(cell.m_j + 1 < grid.JJ())) {
const auto &neighbor = grid.get_cell(cell.m_i, cell.m_j + 1);
cell.populate_neighbor(Loc::B, map, neighbor);
}
if (mode == Mode::PROCESSOR) {
if (cell.processor_boundary(Loc::L) && (cell.rank(Loc::TL) == cell.rank(Loc::C))) {
const auto &tl_corner = grid.get_cell(cell.m_i - 1, cell.m_j + 1);
cell.populate_neighbor(Loc::BR, map, tl_corner);
}
// Now the other "corner case"
if (cell.processor_boundary(Loc::R) && (cell.rank(Loc::TR) == cell.rank(Loc::C))) {
const auto &tr_corner = grid.get_cell(cell.m_i + 1, cell.m_j + 1);
cell.populate_neighbor(Loc::BL, map, tr_corner);
}
}
}
return map;
}
Ioss::PropertyManager parse_properties(SystemInterface &interFace, int int_size)
{
Ioss::PropertyManager properties;
if (int_size == 8) {
properties.add(Ioss::Property("INTEGER_SIZE_DB", 8));
properties.add(Ioss::Property("INTEGER_SIZE_API", 8));
}
if (interFace.use_netcdf4()) {
properties.add(Ioss::Property("FILE_TYPE", "netcdf4"));
}
if (interFace.use_netcdf5()) {
properties.add(Ioss::Property("FILE_TYPE", "netcdf5"));
}
if (interFace.compression_level() > 0 || interFace.szip()) {
properties.add(Ioss::Property("FILE_TYPE", "netcdf4"));
properties.add(Ioss::Property("COMPRESSION_LEVEL", interFace.compression_level()));
properties.add(Ioss::Property("COMPRESSION_SHUFFLE", true));
if (interFace.szip()) {
properties.add(Ioss::Property("COMPRESSION_METHOD", "szip"));
}
else if (interFace.zlib()) {
properties.add(Ioss::Property("COMPRESSION_METHOD", "zlib"));
}
}
return properties;
}
std::shared_ptr<Ioss::Region> create_input_region(const std::string &key, std::string filename,
bool ints_32_bits)
{
// Check that 'filename' does not contain a starting/ending double quote...
filename.erase(remove(filename.begin(), filename.end(), '\"'), filename.end());
Ioss::DatabaseIO *dbi = Ioss::IOFactory::create("exodus", filename, Ioss::READ_RESTART,
Ioss::ParallelUtils::comm_self());
if (dbi == nullptr || !dbi->ok(true)) {
std::exit(EXIT_FAILURE);
}
dbi->set_surface_split_type(Ioss::SPLIT_BY_DONT_SPLIT);
if (ints_32_bits) {
dbi->set_int_byte_size_api(Ioss::USE_INT32_API);
}
else {
dbi->set_int_byte_size_api(Ioss::USE_INT64_API);
}
// Splitting surfaces by element block makes it easier to transform the input
// element id into the output element ids.
dbi->set_surface_split_type(Ioss::SPLIT_BY_ELEMENT_BLOCK);
// Generate a name for the region based on the key...
std::string name = "Region_" + key;
// NOTE: region owns database pointer at this time...
return std::make_shared<Ioss::Region>(dbi, name);
}
void output_summary(Ioss::Region *region, std::ostream &strm)
{
int64_t nodes = region->get_property("node_count").get_int();
int64_t elements = region->get_property("element_count").get_int();
fmt::print(strm, " Database: {}\tNodes = {} \tElements = {}\n",
region->get_database()->get_filename(), fmt::group_digits(nodes),
fmt::group_digits(elements));
}
} // namespace